use std::collections::{BTreeMap, HashMap, VecDeque};
use std::sync::atomic::{
AtomicBool, AtomicI32, AtomicI64, AtomicU32, AtomicU64, AtomicUsize, Ordering,
};
use std::sync::{Arc, Condvar, Mutex, RwLock};
use std::thread::{self, JoinHandle};
use std::time::Instant;
pub use llvm_native_core::openmp::openmp_loop::OpenMPLoopLowering;
pub use llvm_native_core::openmp::openmp_simd::OpenMPSIMDLowering;
pub use llvm_native_core::openmp::{
MapType, MappedVariable, OMPRuntimeCall, OpenMPParallelLowering, OpenMPRuntime,
TargetDataDirective, TargetDevice, TargetOffloadEngine, TargetRegion, OMP_DEFAULT_CHUNK_SIZE,
OMP_MAX_THREADS,
};
pub use llvm_native_core::x86::{
X86ArgClass, X86CallingConvention, X86FrameLowering, X86InstrInfo, X86RegisterInfo,
X86Subtarget, X86TargetMachine, X86_ENDIANNESS, X86_PAGE_SIZE, X86_STACK_ALIGNMENT_64,
};
pub const OMP_MAX_THREADS_X86: u32 = 256;
pub const OMP_DEFAULT_NUM_THREADS: u32 = 0;
pub const OMP_MAX_ACTIVE_LEVELS: u32 = 16;
pub const OMP_MAX_NEST_LOCK_DEPTH: u32 = 256;
pub const OMP_DEFAULT_CHUNK_SIZE_X86: i64 = 1;
pub const OMP_DEFAULT_STACK_SIZE: usize = 8 * 1024 * 1024;
pub const OMP_MAX_TEAMS: u32 = 128;
pub const OMP_MAX_THREADS_PER_TEAM: u32 = 256;
pub const OMP_GUIDED_MIN_CHUNK: i64 = 1;
pub const OMP_DEFAULT_GRAINSIZE: i64 = 0;
pub const OMP_DEFAULT_NUM_TASKS: i64 = 0;
pub const BARRIER_PATTERN_PARALLEL: u32 = 0;
pub const BARRIER_PATTERN_FOR: u32 = 1;
pub const BARRIER_PATTERN_SECTIONS: u32 = 2;
pub const BARRIER_PATTERN_SINGLE: u32 = 3;
pub const BARRIER_PATTERN_TASKGROUP: u32 = 4;
pub const BARRIER_PATTERN_REDUCTION: u32 = 5;
pub const TASK_FLAG_UNTIED: i32 = 0x0001;
pub const TASK_FLAG_FINAL: i32 = 0x0002;
pub const TASK_FLAG_MERGABLE: i32 = 0x0004;
pub const TASK_FLAG_MERGED: i32 = 0x0008;
pub const TASK_FLAG_IMPLICIT: i32 = 0x0010;
pub const TASK_FLAG_TARGET: i32 = 0x0100;
pub const TASK_FLAG_DESTRUCTOR: i32 = 0x0200;
pub const TASK_FLAG_DETACHABLE: i32 = 0x0400;
pub const TASK_FLAG_PRIORITY: i32 = 0x0800;
pub const TASK_FLAG_NONDEFERRED: i32 = 0x1000;
pub const DEPEND_IN: u8 = 1;
pub const DEPEND_OUT: u8 = 2;
pub const DEPEND_INOUT: u8 = 3;
pub const DEPEND_MUTEXINOUTSET: u8 = 4;
pub const DEPEND_SOURCE: u8 = 5;
pub const DEPEND_SINK: u8 = 6;
pub const DEPEND_DEPOBJ: u8 = 7;
pub mod schedule {
pub const SCHEDULE_STATIC: i32 = 1;
pub const SCHEDULE_DYNAMIC: i32 = 2;
pub const SCHEDULE_GUIDED: i32 = 3;
pub const SCHEDULE_AUTO: i32 = 4;
pub const SCHEDULE_RUNTIME: i32 = 5;
pub const SCHEDULE_MONOTONIC: i32 = 0x80000000u32 as i32;
pub const SCHEDULE_NONMONOTONIC: i32 = 0x40000000u32 as i32;
pub fn base_schedule(sched: i32) -> i32 {
sched & !(SCHEDULE_MONOTONIC | SCHEDULE_NONMONOTONIC)
}
pub fn is_monotonic(sched: i32) -> bool {
sched & SCHEDULE_MONOTONIC != 0
}
pub fn is_nonmonotonic(sched: i32) -> bool {
sched & SCHEDULE_NONMONOTONIC != 0
}
}
#[derive(Debug, Clone)]
#[repr(C)]
pub struct Ident {
pub reserved_1: i32,
pub flags: i32,
pub reserved_2: i32,
pub reserved_3: i32,
pub source_loc: *const u8,
}
impl Default for Ident {
fn default() -> Self {
Ident {
reserved_1: 0,
flags: 0,
reserved_2: 0,
reserved_3: 0,
source_loc: std::ptr::null(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(i32)]
pub enum ProcBind {
False = 0,
True = 1,
Master = 2,
Close = 3,
Spread = 4,
}
pub const CANCEL_PARALLEL: i32 = 1;
pub const CANCEL_LOOP: i32 = 2;
pub const CANCEL_SECTIONS: i32 = 4;
pub const CANCEL_TASKGROUP: i32 = 8;
#[derive(Debug, Clone)]
pub struct ICVStorage {
pub num_threads: u32,
pub thread_limit: u32,
pub dynamic: bool,
pub nested: bool,
pub schedule_kind: i32,
pub schedule_chunk: i64,
pub max_active_levels: u32,
pub in_parallel: bool,
pub active_level: u32,
pub place_partition: Vec<i32>,
pub proc_bind: ProcBind,
pub cancellation: bool,
pub stack_size: usize,
pub wait_policy: WaitPolicy,
pub default_schedule: i32,
pub default_chunk: i64,
pub default_device: i32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WaitPolicy {
Active,
Passive,
}
impl Default for ICVStorage {
fn default() -> Self {
ICVStorage {
num_threads: OMP_DEFAULT_NUM_THREADS,
thread_limit: OMP_MAX_THREADS_X86,
dynamic: false,
nested: false,
schedule_kind: schedule::SCHEDULE_STATIC,
schedule_chunk: 0,
max_active_levels: OMP_MAX_ACTIVE_LEVELS,
in_parallel: false,
active_level: 0,
place_partition: Vec::new(),
proc_bind: ProcBind::False,
cancellation: false,
stack_size: OMP_DEFAULT_STACK_SIZE,
wait_policy: WaitPolicy::Active,
default_schedule: schedule::SCHEDULE_STATIC,
default_chunk: 0,
default_device: 0,
}
}
}
impl ICVStorage {
pub fn new() -> Self {
let mut icv = ICVStorage::default();
icv.num_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1) as u32;
icv.thread_limit = (std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 4)
.max(1) as u32;
icv
}
pub fn inherit(&self) -> Self {
ICVStorage {
num_threads: self.num_threads,
thread_limit: self.thread_limit,
dynamic: self.dynamic,
nested: self.nested,
schedule_kind: self.schedule_kind,
schedule_chunk: self.schedule_chunk,
max_active_levels: self.max_active_levels,
in_parallel: true,
active_level: self.active_level + 1,
place_partition: self.place_partition.clone(),
proc_bind: self.proc_bind,
cancellation: self.cancellation,
stack_size: self.stack_size,
wait_policy: self.wait_policy,
default_schedule: self.default_schedule,
default_chunk: self.default_chunk,
default_device: self.default_device,
}
}
pub fn effective_num_threads(&self, num_procs: u32) -> u32 {
let requested = if self.num_threads > 0 {
self.num_threads
} else {
num_procs
};
let limited = requested.min(self.thread_limit);
if self.dynamic {
limited.min(num_procs)
} else {
limited
}
}
pub fn effective_schedule(&self) -> (i32, i64) {
let base = schedule::base_schedule(self.schedule_kind);
if base == schedule::SCHEDULE_RUNTIME {
(self.default_schedule, self.default_chunk)
} else {
(self.schedule_kind, self.schedule_chunk)
}
}
}
pub fn omp_parse_environment(icv: &mut ICVStorage) {
if let Ok(val) = std::env::var("OMP_NUM_THREADS") {
if let Ok(n) = val.parse::<u32>() {
if n > 0 {
icv.num_threads = n;
}
}
}
if let Ok(val) = std::env::var("OMP_THREAD_LIMIT") {
if let Ok(n) = val.parse::<u32>() {
if n > 0 {
icv.thread_limit = n;
}
}
}
if let Ok(val) = std::env::var("OMP_DYNAMIC") {
icv.dynamic = val.to_lowercase() == "true" || val == "1";
}
if let Ok(val) = std::env::var("OMP_NESTED") {
icv.nested = val.to_lowercase() == "true" || val == "1";
}
if let Ok(val) = std::env::var("OMP_SCHEDULE") {
let (kind, chunk) = parse_schedule_env(&val);
icv.schedule_kind = kind;
icv.schedule_chunk = chunk;
icv.default_schedule = kind;
icv.default_chunk = chunk;
}
if let Ok(val) = std::env::var("OMP_MAX_ACTIVE_LEVELS") {
if let Ok(n) = val.parse::<u32>() {
icv.max_active_levels = n;
}
}
if let Ok(val) = std::env::var("OMP_PROC_BIND") {
icv.proc_bind = match val.to_lowercase().as_str() {
"true" => ProcBind::True,
"false" => ProcBind::False,
"master" => ProcBind::Master,
"close" => ProcBind::Close,
"spread" => ProcBind::Spread,
_ => ProcBind::False,
};
}
if let Ok(val) = std::env::var("OMP_CANCELLATION") {
icv.cancellation = val.to_lowercase() == "true" || val == "1";
}
if let Ok(val) = std::env::var("OMP_STACKSIZE") {
if let Some(size) = parse_size(&val) {
icv.stack_size = size;
}
}
if let Ok(val) = std::env::var("OMP_WAIT_POLICY") {
icv.wait_policy = match val.to_lowercase().as_str() {
"active" => WaitPolicy::Active,
"passive" => WaitPolicy::Passive,
_ => WaitPolicy::Active,
};
}
if let Ok(val) = std::env::var("OMP_DEFAULT_DEVICE") {
if let Ok(n) = val.parse::<i32>() {
icv.default_device = n;
}
}
}
fn parse_schedule_env(s: &str) -> (i32, i64) {
let mut parts = s.splitn(2, ',');
let kind_str = parts.next().unwrap_or("static").trim().to_lowercase();
let chunk_str = parts.next().unwrap_or("").trim();
let chunk: i64 = chunk_str.parse().unwrap_or(0);
let kind = match kind_str.as_str() {
"static" => schedule::SCHEDULE_STATIC,
"dynamic" => schedule::SCHEDULE_DYNAMIC,
"guided" => schedule::SCHEDULE_GUIDED,
"auto" => schedule::SCHEDULE_AUTO,
"runtime" => schedule::SCHEDULE_RUNTIME,
_ => schedule::SCHEDULE_STATIC,
};
(kind, chunk)
}
fn parse_size(s: &str) -> Option<usize> {
let s = s.trim().to_uppercase();
let multiplier = if s.ends_with('G') {
1024 * 1024 * 1024
} else if s.ends_with('M') {
1024 * 1024
} else if s.ends_with('K') {
1024
} else {
1
};
let num_str: String = s.chars().take_while(|c| c.is_ascii_digit()).collect();
let num: usize = num_str.parse().ok()?;
Some(num * multiplier)
}
std::thread_local! {
static THREAD_GLOBAL_ID: std::cell::Cell<i32> = const { std::cell::Cell::new(-1) };
static THREAD_ICV: std::cell::RefCell<ICVStorage> = std::cell::RefCell::new(ICVStorage::new());
static THREAD_RUNTIME: std::cell::RefCell<Option<*const X86OpenMP>> = const { std::cell::RefCell::new(None) };
static THREAD_TASK_STATE: std::cell::RefCell<TaskExecutionState> =
std::cell::RefCell::new(TaskExecutionState::new());
static FORK_MICROTASK: std::cell::Cell<Option<unsafe extern "C" fn(i32, i32)>> =
const { std::cell::Cell::new(None) };
}
#[derive(Debug, Clone)]
pub struct TaskExecutionState {
pub current_task_id: u64,
pub in_taskgroup: bool,
pub taskgroup_depth: u32,
pub tasks_created: u64,
pub icv_stack: Vec<ICVStorage>,
}
impl TaskExecutionState {
pub fn new() -> Self {
TaskExecutionState {
current_task_id: 0,
in_taskgroup: false,
taskgroup_depth: 0,
tasks_created: 0,
icv_stack: Vec::new(),
}
}
}
impl Default for TaskExecutionState {
fn default() -> Self {
TaskExecutionState::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(i32)]
pub enum ReductionOp {
Add = 0,
Mul = 1,
And = 2,
Or = 3,
Xor = 4,
BAnd = 5,
BOr = 6,
Max = 7,
Min = 8,
Custom = 9,
}
pub struct ReductionTypeInfo {
pub type_size: usize,
pub op: ReductionOp,
pub init_fn: fn(*mut u8),
pub combine_fn: fn(*mut u8, *const u8),
pub finalize_fn: Option<fn(*mut u8)>,
}
impl Clone for ReductionTypeInfo {
fn clone(&self) -> Self {
ReductionTypeInfo {
type_size: self.type_size,
op: self.op,
init_fn: self.init_fn,
combine_fn: self.combine_fn,
finalize_fn: self.finalize_fn,
}
}
}
impl std::fmt::Debug for ReductionTypeInfo {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ReductionTypeInfo")
.field("type_size", &self.type_size)
.field("op", &self.op)
.finish()
}
}
#[derive(Clone)]
pub struct ReductionEntry {
pub type_info: ReductionTypeInfo,
pub data_ptr: *mut u8,
pub size: usize,
pub owner_gtid: i32,
}
pub struct ReductionContext {
pub entries: Mutex<Vec<ReductionEntry>>,
pub lock: Mutex<()>,
pub active: AtomicBool,
pub generation: AtomicU64,
}
impl ReductionContext {
pub fn new() -> Self {
ReductionContext {
entries: Mutex::new(Vec::new()),
lock: Mutex::new(()),
active: AtomicBool::new(false),
generation: AtomicU64::new(0),
}
}
}
pub struct ReductionTable {
pub table: HashMap<String, ReductionTypeInfo>,
}
impl ReductionTable {
pub fn new() -> Self {
let mut table = HashMap::new();
table.insert(
"i32_add".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) += *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_mul".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Mul,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = 1 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) *= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_and".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::And,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = !0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) &= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_or".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Or,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) |= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_xor".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Xor,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) ^= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_band".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::BAnd,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = !0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) &= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_bor".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::BOr,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i32) |= *(src as *const i32) },
finalize_fn: None,
},
);
table.insert(
"i32_min".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = i32::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i32);
let d = dst as *mut i32;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i32_max".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut i32) = i32::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i32);
let d = dst as *mut i32;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i64_add".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i64) += *(src as *const i64) },
finalize_fn: None,
},
);
table.insert(
"i64_mul".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Mul,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = 1 },
combine_fn: |dst, src| unsafe { *(dst as *mut i64) *= *(src as *const i64) },
finalize_fn: None,
},
);
table.insert(
"i64_and".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::And,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = !0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i64) &= *(src as *const i64) },
finalize_fn: None,
},
);
table.insert(
"i64_or".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Or,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i64) |= *(src as *const i64) },
finalize_fn: None,
},
);
table.insert(
"i64_min".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = i64::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i64);
let d = dst as *mut i64;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i64_max".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut i64) = i64::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i64);
let d = dst as *mut i64;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"f32_add".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut f32) = 0.0 },
combine_fn: |dst, src| unsafe { *(dst as *mut f32) += *(src as *const f32) },
finalize_fn: None,
},
);
table.insert(
"f32_mul".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Mul,
init_fn: |ptr| unsafe { *(ptr as *mut f32) = 1.0 },
combine_fn: |dst, src| unsafe { *(dst as *mut f32) *= *(src as *const f32) },
finalize_fn: None,
},
);
table.insert(
"f32_min".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut f32) = f32::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const f32);
let d = dst as *mut f32;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"f32_max".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut f32) = f32::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const f32);
let d = dst as *mut f32;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"f64_add".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut f64) = 0.0 },
combine_fn: |dst, src| unsafe { *(dst as *mut f64) += *(src as *const f64) },
finalize_fn: None,
},
);
table.insert(
"f64_mul".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Mul,
init_fn: |ptr| unsafe { *(ptr as *mut f64) = 1.0 },
combine_fn: |dst, src| unsafe { *(dst as *mut f64) *= *(src as *const f64) },
finalize_fn: None,
},
);
table.insert(
"f64_min".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut f64) = f64::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const f64);
let d = dst as *mut f64;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"f64_max".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut f64) = f64::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const f64);
let d = dst as *mut f64;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"u32_add".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut u32) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut u32) += *(src as *const u32) },
finalize_fn: None,
},
);
table.insert(
"u32_min".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut u32) = u32::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const u32);
let d = dst as *mut u32;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"u32_max".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut u32) = u32::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const u32);
let d = dst as *mut u32;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"u64_add".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut u64) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut u64) += *(src as *const u64) },
finalize_fn: None,
},
);
table.insert(
"u64_min".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut u64) = u64::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const u64);
let d = dst as *mut u64;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"u64_max".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut u64) = u64::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const u64);
let d = dst as *mut u64;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i8_add".into(),
ReductionTypeInfo {
type_size: 1,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut i8) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i8) += *(src as *const i8) },
finalize_fn: None,
},
);
table.insert(
"i8_min".into(),
ReductionTypeInfo {
type_size: 1,
op: ReductionOp::Min,
init_fn: |ptr| unsafe { *(ptr as *mut i8) = i8::MAX },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i8);
let d = dst as *mut i8;
if s < *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i8_max".into(),
ReductionTypeInfo {
type_size: 1,
op: ReductionOp::Max,
init_fn: |ptr| unsafe { *(ptr as *mut i8) = i8::MIN },
combine_fn: |dst, src| unsafe {
let s = *(src as *const i8);
let d = dst as *mut i8;
if s > *d {
*d = s;
}
},
finalize_fn: None,
},
);
table.insert(
"i16_add".into(),
ReductionTypeInfo {
type_size: 2,
op: ReductionOp::Add,
init_fn: |ptr| unsafe { *(ptr as *mut i16) = 0 },
combine_fn: |dst, src| unsafe { *(dst as *mut i16) += *(src as *const i16) },
finalize_fn: None,
},
);
ReductionTable { table }
}
pub fn get(&self, key: &str) -> Option<&ReductionTypeInfo> {
self.table.get(key)
}
pub fn register(&mut self, key: String, info: ReductionTypeInfo) {
self.table.insert(key, info);
}
pub fn unregister(&mut self, key: &str) -> Option<ReductionTypeInfo> {
self.table.remove(key)
}
pub fn keys(&self) -> Vec<&String> {
self.table.keys().collect()
}
pub fn len(&self) -> usize {
self.table.len()
}
pub fn is_empty(&self) -> bool {
self.table.is_empty()
}
}
impl Default for ReductionTable {
fn default() -> Self {
ReductionTable::new()
}
}
#[derive(Debug)]
pub struct OMPLock {
locked: AtomicBool,
owner: AtomicI32,
contention_count: AtomicU64,
spin_count: u32,
}
impl OMPLock {
const MAX_SPINS: u32 = 1000;
pub fn new() -> Self {
OMPLock {
locked: AtomicBool::new(false),
owner: AtomicI32::new(-1),
contention_count: AtomicU64::new(0),
spin_count: Self::MAX_SPINS,
}
}
pub fn set(&self, gtid: i32) {
let mut spins = 0;
loop {
if self
.locked
.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
{
self.owner.store(gtid, Ordering::Release);
return;
}
spins += 1;
if spins < self.spin_count as u64 {
std::hint::spin_loop();
} else {
self.contention_count.fetch_add(1, Ordering::Relaxed);
std::thread::yield_now();
spins = 0;
}
}
}
pub fn unset(&self) {
self.owner.store(-1, Ordering::Release);
self.locked.store(false, Ordering::Release);
}
pub fn test(&self, gtid: i32) -> bool {
if self
.locked
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
{
self.owner.store(gtid, Ordering::Release);
true
} else {
false
}
}
pub fn is_locked(&self) -> bool {
self.locked.load(Ordering::Acquire)
}
pub fn owner(&self) -> i32 {
self.owner.load(Ordering::Acquire)
}
pub fn contention_count(&self) -> u64 {
self.contention_count.load(Ordering::Relaxed)
}
}
impl Default for OMPLock {
fn default() -> Self {
OMPLock::new()
}
}
unsafe impl Send for OMPLock {}
unsafe impl Sync for OMPLock {}
#[derive(Debug)]
pub struct OMPNestLock {
inner: Mutex<NestLockState>,
}
#[derive(Debug)]
struct NestLockState {
owner: i32,
depth: u32,
contention_count: u64,
}
impl OMPNestLock {
pub fn new() -> Self {
OMPNestLock {
inner: Mutex::new(NestLockState {
owner: -1,
depth: 0,
contention_count: 0,
}),
}
}
pub fn set(&self, gtid: i32) {
let mut spins = 0;
loop {
{
let mut state = self.inner.try_lock();
if let Ok(ref mut state) = state {
if state.owner == -1 || state.owner == gtid {
state.owner = gtid;
if state.depth < OMP_MAX_NEST_LOCK_DEPTH {
state.depth += 1;
}
return;
}
}
}
spins += 1;
if spins < 100 {
std::hint::spin_loop();
} else {
std::thread::yield_now();
spins = 0;
}
}
}
pub fn unset(&self) {
let mut state = self.inner.lock().unwrap();
if state.depth > 0 {
state.depth -= 1;
if state.depth == 0 {
state.owner = -1;
}
}
}
pub fn test(&self, gtid: i32) -> bool {
let mut state = self.inner.lock().unwrap();
if state.owner == -1 || state.owner == gtid {
state.owner = gtid;
if state.depth < OMP_MAX_NEST_LOCK_DEPTH {
state.depth += 1;
}
true
} else {
false
}
}
pub fn is_locked(&self) -> bool {
let state = self.inner.lock().unwrap();
state.depth > 0
}
pub fn depth(&self) -> u32 {
let state = self.inner.lock().unwrap();
state.depth
}
}
impl Default for OMPNestLock {
fn default() -> Self {
OMPNestLock::new()
}
}
unsafe impl Send for OMPNestLock {}
unsafe impl Sync for OMPNestLock {}
pub struct CriticalLockTable {
locks: RwLock<HashMap<String, Arc<Mutex<()>>>>,
}
impl CriticalLockTable {
pub fn new() -> Self {
CriticalLockTable {
locks: RwLock::new(HashMap::new()),
}
}
pub fn get_or_create(&self, name: &str) -> Arc<Mutex<()>> {
{
let read = self.locks.read().unwrap();
if let Some(lock) = read.get(name) {
return Arc::clone(lock);
}
}
let mut write = self.locks.write().unwrap();
write
.entry(name.to_string())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone()
}
pub fn remove(&self, name: &str) -> bool {
let mut write = self.locks.write().unwrap();
write.remove(name).is_some()
}
pub fn len(&self) -> usize {
self.locks.read().unwrap().len()
}
}
impl Default for CriticalLockTable {
fn default() -> Self {
CriticalLockTable::new()
}
}
pub struct WorkerThread {
pub handle: Option<JoinHandle<()>>,
pub gtid: i32,
pub index: usize,
pub idle: Arc<AtomicBool>,
pub wake_signal: Arc<(Mutex<bool>, Condvar)>,
pub done_signal: Arc<(Mutex<bool>, Condvar)>,
pub task_fn: Arc<Mutex<Option<unsafe extern "C" fn(i32, i32)>>>,
shutdown: Arc<AtomicBool>,
}
impl WorkerThread {
fn new(gtid: i32, index: usize, shutdown: Arc<AtomicBool>) -> Self {
WorkerThread {
handle: None,
gtid,
index,
idle: Arc::new(AtomicBool::new(true)),
wake_signal: Arc::new((Mutex::new(false), Condvar::new())),
done_signal: Arc::new((Mutex::new(true), Condvar::new())),
task_fn: Arc::new(Mutex::new(None)),
shutdown,
}
}
}
unsafe impl Send for WorkerThread {}
unsafe impl Sync for WorkerThread {}
pub struct ThreadPool {
pub workers: RwLock<Vec<Arc<WorkerThread>>>,
pub num_threads: AtomicU32,
next_gtid: AtomicI32,
shutdown: Arc<AtomicBool>,
wait_policy: RwLock<WaitPolicy>,
stack_size: AtomicUsize,
}
impl ThreadPool {
const MAX_SPINS: u32 = 1000;
pub fn new() -> Self {
ThreadPool {
workers: RwLock::new(Vec::new()),
num_threads: AtomicU32::new(1),
next_gtid: AtomicI32::new(0),
shutdown: Arc::new(AtomicBool::new(false)),
wait_policy: RwLock::new(WaitPolicy::Active),
stack_size: AtomicUsize::new(OMP_DEFAULT_STACK_SIZE),
}
}
pub fn ensure_workers(&self, n: u32) {
let mut workers = self.workers.write().unwrap();
while (workers.len() as u32) < n {
let gtid = self.next_gtid.fetch_add(1, Ordering::SeqCst);
let index = workers.len();
let worker = Arc::new(WorkerThread::new(gtid, index, Arc::clone(&self.shutdown)));
let w = Arc::clone(&worker);
let shutdown = Arc::clone(&self.shutdown);
let policy = *self.wait_policy.read().unwrap();
let stack_sz = self.stack_size.load(Ordering::Acquire);
let mut builder = thread::Builder::new().name(format!("omp-worker-{}", index));
if stack_sz > 0 {
builder = builder.stack_size(stack_sz);
}
let handle = builder
.spawn(move || {
Self::worker_loop(w, shutdown, policy);
})
.expect("Failed to spawn OpenMP worker thread");
unsafe {
let worker_ptr = Arc::as_ptr(&worker) as *mut WorkerThread;
(*worker_ptr).handle = Some(handle);
}
workers.push(worker);
}
self.num_threads.store(n, Ordering::Release);
}
fn worker_loop(worker: Arc<WorkerThread>, shutdown: Arc<AtomicBool>, policy: WaitPolicy) {
THREAD_GLOBAL_ID.with(|id| id.set(worker.gtid));
loop {
if shutdown.load(Ordering::Acquire) {
break;
}
{
let (ref lock, ref cvar) = *worker.wake_signal;
let mut started = lock.lock().unwrap();
if policy == WaitPolicy::Active {
let mut spins = 0;
loop {
if *started || shutdown.load(Ordering::Acquire) {
break;
}
if spins < ThreadPool::MAX_SPINS {
std::hint::spin_loop();
spins += 1;
} else {
started = cvar.wait(started).unwrap();
break;
}
}
} else {
while !*started && !shutdown.load(Ordering::Acquire) {
started = cvar.wait(started).unwrap();
}
}
if shutdown.load(Ordering::Acquire) {
break;
}
*started = false;
}
worker.idle.store(false, Ordering::Release);
if let Some(task_fn) = *worker.task_fn.lock().unwrap() {
unsafe {
task_fn(worker.gtid, 1);
}
}
{
let (ref lock, ref cvar) = *worker.done_signal;
let mut done = lock.lock().unwrap();
*done = true;
cvar.notify_one();
}
worker.idle.store(true, Ordering::Release);
}
}
pub fn fork(&self, task_fn: unsafe extern "C" fn(i32, i32), num_threads: u32) {
self.ensure_workers(num_threads);
let workers = self.workers.read().unwrap();
let n = num_threads.min(workers.len() as u32);
let n_workers = if n > 1 { n - 1 } else { 0 };
for i in 0..n_workers as usize {
*workers[i].task_fn.lock().unwrap() = Some(task_fn);
}
for i in 0..n_workers as usize {
let (ref lock, ref cvar) = *workers[i].wake_signal;
let mut started = lock.lock().unwrap();
*started = true;
cvar.notify_one();
}
for i in 0..n_workers as usize {
let (ref lock, ref cvar) = *workers[i].done_signal;
let mut done = lock.lock().unwrap();
while !*done {
done = cvar.wait(done).unwrap();
}
*done = false;
}
for i in 0..n_workers as usize {
*workers[i].task_fn.lock().unwrap() = None;
}
}
pub fn shutdown(&self) {
self.shutdown.store(true, Ordering::Release);
let workers = self.workers.read().unwrap();
for worker in workers.iter() {
let (ref lock, ref cvar) = *worker.wake_signal;
let mut started = lock.lock().unwrap();
*started = true;
cvar.notify_one();
}
}
pub fn set_wait_policy(&self, policy: WaitPolicy) {
*self.wait_policy.write().unwrap() = policy;
}
pub fn set_stack_size(&self, size: usize) {
self.stack_size.store(size, Ordering::Release);
}
}
impl Default for ThreadPool {
fn default() -> Self {
ThreadPool::new()
}
}
pub struct TaskDescriptor {
pub task_id: u64,
pub task_fn: Option<unsafe extern "C" fn(i32, *mut TaskDescriptor)>,
pub parent_gtid: i32,
pub flags: i32,
pub undeferred: bool,
pub merging: bool,
pub implicit: bool,
pub dep_count: u32,
pub deps_satisfied: bool,
pub shareds: *mut u8,
pub shareds_size: usize,
pub status: AtomicU32,
pub priority: i32,
pub tied: bool,
pub created_at: Instant,
pub estimated_cost: u64,
}
impl TaskDescriptor {
pub fn new(task_id: u64, parent_gtid: i32) -> Self {
TaskDescriptor {
task_id,
task_fn: None,
parent_gtid,
flags: 0,
undeferred: false,
merging: false,
implicit: false,
dep_count: 0,
deps_satisfied: true,
shareds: std::ptr::null_mut(),
shareds_size: 0,
status: AtomicU32::new(0),
priority: 0,
tied: true,
created_at: Instant::now(),
estimated_cost: 0,
}
}
pub fn is_tied(&self) -> bool {
self.tied && !self.is_untied()
}
pub fn is_untied(&self) -> bool {
self.flags & TASK_FLAG_UNTIED != 0
}
pub fn is_final(&self) -> bool {
self.flags & TASK_FLAG_FINAL != 0
}
pub fn is_mergable(&self) -> bool {
self.flags & TASK_FLAG_MERGABLE != 0
}
pub fn is_target_task(&self) -> bool {
self.flags & TASK_FLAG_TARGET != 0
}
}
impl Clone for TaskDescriptor {
fn clone(&self) -> Self {
TaskDescriptor {
task_id: self.task_id,
task_fn: self.task_fn,
parent_gtid: self.parent_gtid,
flags: self.flags,
undeferred: self.undeferred,
merging: self.merging,
implicit: self.implicit,
dep_count: self.dep_count,
deps_satisfied: self.deps_satisfied,
shareds: self.shareds,
shareds_size: self.shareds_size,
status: AtomicU32::new(self.status.load(Ordering::Acquire)),
priority: self.priority,
tied: self.tied,
created_at: self.created_at,
estimated_cost: self.estimated_cost,
}
}
}
unsafe impl Send for TaskDescriptor {}
unsafe impl Sync for TaskDescriptor {}
#[derive(Debug, Clone)]
pub struct TaskDependence {
pub dep_type: u8,
pub addr: usize,
pub size: usize,
}
pub struct TaskQueue {
pub ready_queue: Mutex<VecDeque<Arc<TaskDescriptor>>>,
pub waiting_queue: Mutex<VecDeque<Arc<TaskDescriptor>>>,
pub completed: Mutex<Vec<u64>>,
pub dep_graph: Mutex<BTreeMap<usize, Vec<(u64, u8)>>>,
next_task_id: AtomicU64,
taskgroup_depth: AtomicU32,
taskgroup_tasks: Mutex<Vec<Arc<TaskDescriptor>>>,
total_created: AtomicU64,
total_completed: AtomicU64,
}
impl TaskQueue {
pub fn new() -> Self {
TaskQueue {
ready_queue: Mutex::new(VecDeque::new()),
waiting_queue: Mutex::new(VecDeque::new()),
completed: Mutex::new(Vec::new()),
dep_graph: Mutex::new(BTreeMap::new()),
next_task_id: AtomicU64::new(0),
taskgroup_depth: AtomicU32::new(0),
taskgroup_tasks: Mutex::new(Vec::new()),
total_created: AtomicU64::new(0),
total_completed: AtomicU64::new(0),
}
}
pub fn allocate_task_id(&self) -> u64 {
self.next_task_id.fetch_add(1, Ordering::SeqCst)
}
pub fn enqueue_task(&self, task: Arc<TaskDescriptor>) {
self.total_created.fetch_add(1, Ordering::Relaxed);
if !task.deps_satisfied {
let mut waiting = self.waiting_queue.lock().unwrap();
waiting.push_back(task);
return;
}
let mut queue = self.ready_queue.lock().unwrap();
if task.priority > 0 {
let pos = queue.iter().position(|t| t.priority < task.priority);
match pos {
Some(idx) => queue.insert(idx, task),
None => queue.push_back(task),
}
} else {
queue.push_back(task);
}
}
pub fn dequeue_task(&self) -> Option<Arc<TaskDescriptor>> {
let mut queue = self.ready_queue.lock().unwrap();
queue.pop_front()
}
pub fn steal_task(&self) -> Option<Arc<TaskDescriptor>> {
let mut queue = self.ready_queue.lock().unwrap();
queue.pop_back()
}
pub fn taskwait(&self, gtid: i32) {
loop {
let task = self.dequeue_task();
match task {
Some(t) => {
self.execute_task(&t, gtid);
}
None => {
self.resolve_dependences();
let task = self.dequeue_task();
match task {
Some(t) => self.execute_task(&t, gtid),
None => break,
}
}
}
}
}
fn execute_task(&self, task: &Arc<TaskDescriptor>, gtid: i32) {
if let Some(f) = task.task_fn {
task.status.store(1, Ordering::Release);
unsafe {
f(gtid, Arc::as_ptr(task) as *mut TaskDescriptor);
}
task.status.store(2, Ordering::Release);
self.total_completed.fetch_add(1, Ordering::Relaxed);
self.completed.lock().unwrap().push(task.task_id);
}
}
fn resolve_dependences(&self) {
let mut waiting = self.waiting_queue.lock().unwrap();
let mut ready = Vec::new();
let mut still_waiting = VecDeque::new();
while let Some(task) = waiting.pop_front() {
let all_satisfied = true; if all_satisfied {
ready.push(task);
} else {
still_waiting.push_back(task);
}
}
*waiting = still_waiting;
if !ready.is_empty() {
let mut rq = self.ready_queue.lock().unwrap();
for task in ready {
rq.push_back(task);
}
}
}
pub fn begin_taskgroup(&self) {
self.taskgroup_depth.fetch_add(1, Ordering::SeqCst);
}
pub fn end_taskgroup(&self, gtid: i32) {
let depth = self.taskgroup_depth.fetch_sub(1, Ordering::SeqCst);
if depth == 1 {
self.taskwait(gtid);
let tasks = {
let mut tg = self.taskgroup_tasks.lock().unwrap();
std::mem::take(&mut *tg)
};
for task in tasks {
loop {
let status = task.status.load(Ordering::Acquire);
if status == 2 || status == 3 {
break;
}
self.execute_task(&task, gtid);
}
}
}
}
pub fn add_to_taskgroup(&self, task: Arc<TaskDescriptor>) {
let depth = self.taskgroup_depth.load(Ordering::Acquire);
if depth > 0 {
let mut tg = self.taskgroup_tasks.lock().unwrap();
tg.push(task);
}
}
pub fn total_created(&self) -> u64 {
self.total_created.load(Ordering::Relaxed)
}
pub fn total_completed(&self) -> u64 {
self.total_completed.load(Ordering::Relaxed)
}
pub fn ready_count(&self) -> usize {
self.ready_queue.lock().unwrap().len()
}
pub fn waiting_count(&self) -> usize {
self.waiting_queue.lock().unwrap().len()
}
}
impl Default for TaskQueue {
fn default() -> Self {
TaskQueue::new()
}
}
#[derive(Debug, Clone)]
pub struct WorkshareState {
pub lower: i64,
pub upper: i64,
pub stride: i64,
pub chunk: i64,
pub last_iter: i32,
pub num_threads: u32,
pub schedule: i32,
pub next_iter: i64,
pub iterations_remaining: i64,
pub initialized: bool,
pub incr: i64,
pub is_less_than_bound: bool,
}
impl Default for WorkshareState {
fn default() -> Self {
WorkshareState {
lower: 0,
upper: 0,
stride: 1,
chunk: 1,
last_iter: 0,
num_threads: 1,
schedule: schedule::SCHEDULE_STATIC,
next_iter: 0,
iterations_remaining: 0,
initialized: false,
incr: 1,
is_less_than_bound: true,
}
}
}
pub struct TeamWorkshare {
pub state: Mutex<WorkshareState>,
pub thread_bounds: Mutex<Vec<(i64, i64)>>,
pub active: AtomicBool,
}
impl TeamWorkshare {
pub fn new() -> Self {
TeamWorkshare {
state: Mutex::new(WorkshareState::default()),
thread_bounds: Mutex::new(Vec::new()),
active: AtomicBool::new(false),
}
}
pub fn reset(&self) {
let mut state = self.state.lock().unwrap();
*state = WorkshareState::default();
self.thread_bounds.lock().unwrap().clear();
self.active.store(false, Ordering::Release);
}
}
impl Default for TeamWorkshare {
fn default() -> Self {
TeamWorkshare::new()
}
}
pub struct SectionsState {
pub num_sections: u32,
pub next_section: AtomicU32,
}
impl SectionsState {
pub fn new(num_sections: u32) -> Self {
SectionsState {
num_sections,
next_section: AtomicU32::new(0),
}
}
pub fn next(&self) -> Option<u32> {
let current = self.next_section.fetch_add(1, Ordering::SeqCst);
if current < self.num_sections {
Some(current)
} else {
None
}
}
pub fn reset(&self, num_sections: u32) {
self.next_section.store(0, Ordering::Release);
}
}
pub struct DistForState {
pub chunk_lower: [i64; 64],
pub chunk_upper: [i64; 64],
pub chunk_stride: [i64; 64],
pub num_chunks: u32,
pub schedule: i32,
pub initialized: bool,
}
impl Default for DistForState {
fn default() -> Self {
DistForState {
chunk_lower: [0; 64],
chunk_upper: [0; 64],
chunk_stride: [1; 64],
num_chunks: 0,
schedule: schedule::SCHEDULE_STATIC,
initialized: false,
}
}
}
pub struct TaskLoopState {
pub lower: i64,
pub upper: i64,
pub stride: i64,
pub grainsize: i64,
pub num_tasks: i64,
pub current: AtomicI64,
pub tasks_created: AtomicU32,
}
impl TaskLoopState {
pub fn new(lower: i64, upper: i64, stride: i64, grainsize: i64, num_tasks: i64) -> Self {
TaskLoopState {
lower,
upper,
stride,
grainsize,
num_tasks,
current: AtomicI64::new(lower),
tasks_created: AtomicU32::new(0),
}
}
pub fn chunk_size(&self) -> i64 {
if self.grainsize > 0 {
self.grainsize
} else if self.num_tasks > 0 {
let total = (self.upper - self.lower) / self.stride + 1;
(total / self.num_tasks).max(1)
} else {
1
}
}
pub fn next_chunk(&self) -> Option<(i64, i64)> {
let chunk_size = self.chunk_size();
loop {
let current = self.current.load(Ordering::SeqCst);
if current > self.upper {
return None;
}
let next = (current + chunk_size * self.stride).min(self.upper + self.stride);
if self
.current
.compare_exchange(current, next, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
self.tasks_created.fetch_add(1, Ordering::Relaxed);
let end = (next - self.stride).min(self.upper);
return Some((current, end));
}
}
}
pub fn has_more(&self) -> bool {
self.current.load(Ordering::SeqCst) <= self.upper
}
}
pub struct OrderedState {
pub next_ordered: AtomicI64,
pub in_ordered: AtomicBool,
}
impl OrderedState {
pub fn new() -> Self {
OrderedState {
next_ordered: AtomicI64::new(0),
in_ordered: AtomicBool::new(false),
}
}
}
impl Default for OrderedState {
fn default() -> Self {
OrderedState::new()
}
}
pub struct TeamBarrier {
pub count: AtomicU32,
pub arrived: AtomicU32,
pub generation: AtomicU32,
pub cancel_flag: AtomicBool,
pub condvar: Condvar,
pub mutex: Mutex<u32>,
pub pattern: AtomicU32,
pub usage_count: AtomicU64,
}
impl TeamBarrier {
pub fn new(num_threads: u32) -> Self {
TeamBarrier {
count: AtomicU32::new(num_threads),
arrived: AtomicU32::new(0),
generation: AtomicU32::new(0),
cancel_flag: AtomicBool::new(false),
condvar: Condvar::new(),
mutex: Mutex::new(0),
pattern: AtomicU32::new(BARRIER_PATTERN_PARALLEL),
usage_count: AtomicU64::new(0),
}
}
pub fn wait(&self) {
self.usage_count.fetch_add(1, Ordering::Relaxed);
let r#gen = self.generation.load(Ordering::Acquire);
let prev = self.arrived.fetch_add(1, Ordering::AcqRel);
if prev + 1 == self.count.load(Ordering::Acquire) {
self.arrived.store(0, Ordering::Release);
self.generation.fetch_add(1, Ordering::AcqRel);
let _lock = self.mutex.lock().unwrap();
self.condvar.notify_all();
} else {
let lock = self.mutex.lock().unwrap();
let mut guard = lock;
loop {
if self.cancel_flag.load(Ordering::Acquire) {
break;
}
if self.generation.load(Ordering::Acquire) != r#gen {
break;
}
guard = self.condvar.wait(guard).unwrap();
}
}
}
pub fn wait_cancellable(&self) -> bool {
self.usage_count.fetch_add(1, Ordering::Relaxed);
let r#gen = self.generation.load(Ordering::Acquire);
let prev = self.arrived.fetch_add(1, Ordering::AcqRel);
if self.cancel_flag.load(Ordering::Acquire) {
return true;
}
if prev + 1 == self.count.load(Ordering::Acquire) {
self.arrived.store(0, Ordering::Release);
self.generation.fetch_add(1, Ordering::AcqRel);
let _lock = self.mutex.lock().unwrap();
self.condvar.notify_all();
false
} else {
let lock = self.mutex.lock().unwrap();
let mut guard = lock;
loop {
if self.cancel_flag.load(Ordering::Acquire) {
return true;
}
if self.generation.load(Ordering::Acquire) != r#gen {
return false;
}
guard = self.condvar.wait(guard).unwrap();
}
}
}
pub fn cancel(&self) {
self.cancel_flag.store(true, Ordering::Release);
let _lock = self.mutex.lock().unwrap();
self.condvar.notify_all();
}
pub fn reset(&self, num_threads: u32) {
self.count.store(num_threads, Ordering::Release);
self.arrived.store(0, Ordering::Release);
self.cancel_flag.store(false, Ordering::Release);
self.pattern
.store(BARRIER_PATTERN_PARALLEL, Ordering::Release);
}
pub fn set_pattern(&self, pattern: u32) {
self.pattern.store(pattern, Ordering::Release);
}
pub fn arrived_count(&self) -> u32 {
self.arrived.load(Ordering::Acquire)
}
pub fn is_cancelled(&self) -> bool {
self.cancel_flag.load(Ordering::Acquire)
}
}
pub struct SingleState {
pub executed: AtomicBool,
}
impl SingleState {
pub fn new() -> Self {
SingleState {
executed: AtomicBool::new(false),
}
}
pub fn try_acquire(&self) -> bool {
self.executed
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
}
pub fn reset(&self) {
self.executed.store(false, Ordering::Release);
}
}
impl Default for SingleState {
fn default() -> Self {
SingleState::new()
}
}
pub struct MasterState {
pub master_gtid: AtomicI32,
}
impl MasterState {
pub fn new(master_gtid: i32) -> Self {
MasterState {
master_gtid: AtomicI32::new(master_gtid),
}
}
pub fn is_master(&self, gtid: i32) -> bool {
self.master_gtid.load(Ordering::Acquire) == gtid
}
}
pub struct ParallelRegionState {
pub master_gtid: i32,
pub num_threads: u32,
pub barrier: Arc<TeamBarrier>,
pub workshare: Arc<TeamWorkshare>,
pub single: Arc<SingleState>,
pub master: Arc<MasterState>,
pub ordered: Arc<OrderedState>,
pub cancelled: AtomicBool,
pub reductions: Mutex<HashMap<usize, Arc<ReductionContext>>>,
pub level: u32,
pub in_teams: bool,
}
impl ParallelRegionState {
pub fn new(master_gtid: i32, num_threads: u32) -> Self {
ParallelRegionState {
master_gtid,
num_threads,
barrier: Arc::new(TeamBarrier::new(num_threads)),
workshare: Arc::new(TeamWorkshare::new()),
single: Arc::new(SingleState::new()),
master: Arc::new(MasterState::new(master_gtid)),
ordered: Arc::new(OrderedState::new()),
cancelled: AtomicBool::new(false),
reductions: Mutex::new(HashMap::new()),
level: 0,
in_teams: false,
}
}
pub fn get_reduction(&self, key: usize) -> Arc<ReductionContext> {
let mut reductions = self.reductions.lock().unwrap();
reductions
.entry(key)
.or_insert_with(|| Arc::new(ReductionContext::new()))
.clone()
}
}
pub struct X86OpenMP {
pub thread_pool: ThreadPool,
pub task_queue: TaskQueue,
pub reduction_table: ReductionTable,
pub critical_locks: CriticalLockTable,
pub lock_table: Mutex<HashMap<usize, Arc<OMPLock>>>,
pub nest_lock_table: Mutex<HashMap<usize, Arc<OMPNestLock>>>,
next_gtid: AtomicI32,
pub num_procs: u32,
start_time: Instant,
initialized: AtomicBool,
shutdown: AtomicBool,
pub parallel_region_stack: Mutex<Vec<Arc<ParallelRegionState>>>,
pub proc_bind: RwLock<ProcBind>,
pub places: RwLock<Vec<Vec<i32>>>,
pub env_parsed: AtomicBool,
pub debug_level: AtomicU32,
}
impl X86OpenMP {
pub fn new() -> Self {
X86OpenMP {
thread_pool: ThreadPool::new(),
task_queue: TaskQueue::new(),
reduction_table: ReductionTable::new(),
critical_locks: CriticalLockTable::new(),
lock_table: Mutex::new(HashMap::new()),
nest_lock_table: Mutex::new(HashMap::new()),
next_gtid: AtomicI32::new(0),
num_procs: std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1) as u32,
start_time: Instant::now(),
initialized: AtomicBool::new(false),
shutdown: AtomicBool::new(false),
parallel_region_stack: Mutex::new(Vec::new()),
proc_bind: RwLock::new(ProcBind::False),
places: RwLock::new(Vec::new()),
env_parsed: AtomicBool::new(false),
debug_level: AtomicU32::new(0),
}
}
pub fn initialize(&self) {
if self
.initialized
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
if !self.env_parsed.load(Ordering::Acquire) {
THREAD_ICV.with(|icv| {
omp_parse_environment(&mut icv.borrow_mut());
});
self.env_parsed.store(true, Ordering::Release);
}
let gtid = self.next_gtid.fetch_add(1, Ordering::SeqCst);
THREAD_GLOBAL_ID.with(|id| id.set(gtid));
{
let mut places = self.places.write().unwrap();
for i in 0..self.num_procs {
places.push(vec![i as i32]);
}
}
}
}
pub fn shutdown(&self) {
self.shutdown.store(true, Ordering::Release);
self.thread_pool.shutdown();
}
pub fn allocate_gtid(&self) -> i32 {
self.next_gtid.fetch_add(1, Ordering::SeqCst)
}
pub fn get_gtid(&self) -> i32 {
THREAD_GLOBAL_ID.with(|id| id.get())
}
pub fn get_num_threads(&self) -> u32 {
THREAD_ICV.with(|icv| {
let icv = icv.borrow();
icv.effective_num_threads(self.num_procs)
})
}
pub fn is_shutting_down(&self) -> bool {
self.shutdown.load(Ordering::Acquire)
}
pub fn is_initialized(&self) -> bool {
self.initialized.load(Ordering::Acquire)
}
pub fn push_parallel_region(&self, mut region: Arc<ParallelRegionState>) {
let mut stack = self.parallel_region_stack.lock().unwrap();
unsafe {
let region_mut = Arc::as_ptr(®ion) as *mut ParallelRegionState;
(*region_mut).level = stack.len() as u32;
}
stack.push(region);
}
pub fn pop_parallel_region(&self) -> Option<Arc<ParallelRegionState>> {
let mut stack = self.parallel_region_stack.lock().unwrap();
stack.pop()
}
pub fn current_parallel_region(&self) -> Option<Arc<ParallelRegionState>> {
let stack = self.parallel_region_stack.lock().unwrap();
stack.last().cloned()
}
pub fn get_active_level(&self) -> u32 {
let stack = self.parallel_region_stack.lock().unwrap();
stack.len() as u32
}
pub fn get_ancestor_thread_num(&self, level: u32) -> i32 {
let stack = self.parallel_region_stack.lock().unwrap();
if level as usize >= stack.len() {
return -1;
}
if level == 0 {
return 0;
}
THREAD_GLOBAL_ID.with(|id| id.get())
}
pub fn get_team_size(&self, level: u32) -> i32 {
let stack = self.parallel_region_stack.lock().unwrap();
if level as usize >= stack.len() {
return -1;
}
stack[level as usize].num_threads as i32
}
pub fn is_master(&self) -> bool {
match self.current_parallel_region() {
Some(region) => {
let gtid = self.get_gtid();
region.master.is_master(gtid)
}
None => true, }
}
pub fn set_debug_level(&self, level: u32) {
self.debug_level.store(level, Ordering::Release);
}
pub fn debug_log(&self, threshold: u32, msg: &str) {
if self.debug_level.load(Ordering::Relaxed) >= threshold {
let gtid = self.get_gtid();
eprintln!("[OMP debug gtid={}] {}", gtid, msg);
}
}
}
impl Default for X86OpenMP {
fn default() -> Self {
X86OpenMP::new()
}
}
unsafe impl Send for X86OpenMP {}
unsafe impl Sync for X86OpenMP {}
pub static GLOBAL_RUNTIME: std::sync::LazyLock<Arc<X86OpenMP>> = std::sync::LazyLock::new(|| {
let runtime = Arc::new(X86OpenMP::new());
runtime.initialize();
runtime
});
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_fork_call(
_loc: *const Ident,
_argc: i32,
microtask: unsafe extern "C" fn(i32, i32),
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let master_gtid = runtime.get_gtid();
let num_threads = runtime.get_num_threads();
let actual_threads = num_threads.min(OMP_MAX_THREADS_X86).max(1);
let can_nest = THREAD_ICV.with(|icv| {
let icv = icv.borrow();
icv.nested && icv.active_level < icv.max_active_levels
});
let actual_threads = if can_nest || runtime.get_active_level() == 0 {
actual_threads
} else {
1 };
let region = Arc::new(ParallelRegionState::new(master_gtid, actual_threads));
runtime.push_parallel_region(Arc::clone(®ion));
THREAD_ICV.with(|icv| {
let mut icv = icv.borrow_mut();
icv.in_parallel = true;
icv.active_level += 1;
});
runtime.debug_log(
1,
&format!(
"fork_call: {} threads, master_gtid={}",
actual_threads, master_gtid
),
);
if actual_threads > 1 {
let task = microtask;
runtime.thread_pool.fork(
{
unsafe extern "C" fn wrapper(gtid: i32, _bound_tid: i32) {
THREAD_GLOBAL_ID.with(|id| id.set(gtid));
THREAD_ICV.with(|icv| {
let mut icv = icv.borrow_mut();
icv.in_parallel = true;
icv.active_level += 1;
});
FORK_MICROTASK.with(|cell| {
if let Some(func) = cell.get() {
unsafe {
func(gtid, 0);
}
}
});
THREAD_ICV.with(|icv| {
let mut icv = icv.borrow_mut();
icv.in_parallel = false;
icv.active_level -= 1;
});
}
wrapper
},
actual_threads - 1,
);
FORK_MICROTASK.with(|cell| cell.set(Some(task)));
microtask(master_gtid, 0);
FORK_MICROTASK.with(|cell| cell.set(None));
} else {
microtask(master_gtid, 0);
}
region.barrier.wait();
THREAD_ICV.with(|icv| {
let mut icv = icv.borrow_mut();
icv.in_parallel = false;
icv.active_level -= 1;
});
runtime.pop_parallel_region();
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_push_num_threads(_loc: *const Ident, _gtid: i32, num_threads: i32) {
if num_threads > 0 {
THREAD_ICV.with(|icv| {
icv.borrow_mut().num_threads = num_threads as u32;
});
}
}
fn compute_static_bounds(
total_iters: i64,
chunk_size: i64,
num_threads: u32,
tid: u32,
base: i64,
incr: i64,
upper: i64,
) -> (i64, i64, bool) {
let num_chunks = (total_iters + chunk_size - 1) / chunk_size;
if num_chunks == 0 {
return (0, 0, false);
}
let chunks_per_thread = num_chunks / num_threads as i64;
let remainder = num_chunks % num_threads as i64;
let (start_chunk, end_chunk) = if (tid as i64) < remainder {
let start = tid as i64 * (chunks_per_thread + 1);
(start, start + chunks_per_thread)
} else {
let start =
remainder * (chunks_per_thread + 1) + (tid as i64 - remainder) * chunks_per_thread;
(start, start + chunks_per_thread - 1)
};
if end_chunk >= start_chunk && start_chunk < num_chunks {
let thread_lower = base + start_chunk * chunk_size * incr;
let thread_upper =
(base + end_chunk * chunk_size * incr + (chunk_size - 1) * incr).min(upper);
let is_last = end_chunk == num_chunks - 1;
(thread_lower, thread_upper, is_last)
} else {
(0, 0, false)
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_4(
_loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut i32,
pupper: *mut i32,
pstride: *mut i32,
incr: i32,
chunk: i32,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower as i64;
let upper = *pupper as i64;
let stride = *pstride as i64;
let effective_chunk = if chunk == 0 { 1i64 } else { chunk as i64 };
let incr_i64 = incr as i64;
let num_threads = match runtime.current_parallel_region() {
Some(region) => region.num_threads,
None => 1,
};
let tid = (gtid % num_threads as i32).max(0) as u32;
let base_sched = schedule::base_schedule(sched);
match base_sched {
schedule::SCHEDULE_STATIC => {
let total_iters = if stride != 0 {
if stride > 0 {
((upper - lower) / incr_i64).abs() + 1
} else {
((lower - upper) / (-incr_i64)).abs() + 1
}
} else {
0
};
let (thread_lower, thread_upper, is_last) = compute_static_bounds(
total_iters,
effective_chunk,
num_threads,
tid,
lower,
incr_i64,
upper,
);
*plower = thread_lower as i32;
*pupper = thread_upper as i32;
*pstride = incr;
*plastiter = if is_last { 1 } else { 0 };
}
schedule::SCHEDULE_DYNAMIC | schedule::SCHEDULE_GUIDED => {
let total_iters = if stride != 0 {
if stride > 0 {
((upper - lower) / incr_i64).abs() + 1
} else {
((lower - upper) / (-incr_i64)).abs() + 1
}
} else {
0
};
if total_iters > 0 {
let chunk_size = effective_chunk * incr_i64;
let thread_lower = lower + tid as i64 * chunk_size;
let thread_upper = (thread_lower + chunk_size - incr_i64).min(upper);
*plower = thread_lower as i32;
*pupper = thread_upper as i32;
*pstride = incr;
*plastiter = 0;
} else {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
_ => {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_8(
_loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut i64,
pupper: *mut i64,
pstride: *mut i64,
incr: i64,
chunk: i64,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower;
let upper = *pupper;
let stride = *pstride;
let effective_chunk = if chunk == 0 { 1 } else { chunk };
let num_threads = match runtime.current_parallel_region() {
Some(region) => region.num_threads,
None => 1,
};
let tid = (gtid % num_threads as i32).max(0) as u32;
let base_sched = schedule::base_schedule(sched);
match base_sched {
schedule::SCHEDULE_STATIC => {
let total_iters = if stride != 0 {
if stride > 0 {
((upper - lower) / incr).abs() + 1
} else {
((lower - upper) / (-incr)).abs() + 1
}
} else {
0
};
let (thread_lower, thread_upper, is_last) = compute_static_bounds(
total_iters,
effective_chunk,
num_threads,
tid,
lower,
incr,
upper,
);
*plower = thread_lower;
*pupper = thread_upper;
*pstride = incr;
*plastiter = if is_last { 1 } else { 0 };
}
schedule::SCHEDULE_DYNAMIC | schedule::SCHEDULE_GUIDED => {
let total_iters = if stride != 0 {
if stride > 0 {
((upper - lower) / incr).abs() + 1
} else {
((lower - upper) / (-incr)).abs() + 1
}
} else {
0
};
if total_iters > 0 {
let chunk_size = effective_chunk * incr;
let thread_lower = lower + tid as i64 * chunk_size;
let thread_upper = (thread_lower + chunk_size - incr).min(upper);
*plower = thread_lower;
*pupper = thread_upper;
*pstride = incr;
*plastiter = 0;
} else {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
_ => {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_4u(
_loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut u32,
pupper: *mut u32,
pstride: *mut i32,
incr: i32,
chunk: i32,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower as i64;
let upper = *pupper as i64;
let effective_chunk = if chunk == 0 { 1i64 } else { chunk as i64 };
let incr_i64 = incr as i64;
let num_threads = match runtime.current_parallel_region() {
Some(region) => region.num_threads,
None => 1,
};
let tid = (gtid % num_threads as i32).max(0) as u32;
match schedule::base_schedule(sched) {
schedule::SCHEDULE_STATIC => {
let total_iters = (upper - lower) / incr_i64.abs().max(1) + 1;
let (thread_lower, thread_upper, is_last) = compute_static_bounds(
total_iters,
effective_chunk,
num_threads,
tid,
lower,
incr_i64,
upper,
);
*plower = thread_lower.max(0) as u32;
*pupper = thread_upper.max(0) as u32;
*pstride = incr;
*plastiter = if is_last { 1 } else { 0 };
}
_ => {
let total_iters = (upper - lower) / incr_i64.abs().max(1) + 1;
if total_iters > 0 {
let chunk_size = effective_chunk * incr_i64;
let thread_lower = lower + tid as i64 * chunk_size;
let thread_upper = (thread_lower + chunk_size - incr_i64).min(upper);
*plower = thread_lower.max(0) as u32;
*pupper = thread_upper.max(0) as u32;
*pstride = incr;
*plastiter = 0;
} else {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_8u(
_loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut u64,
pupper: *mut u64,
pstride: *mut i64,
incr: i64,
chunk: i64,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower as i64;
let upper = *pupper as i64;
let effective_chunk = if chunk == 0 { 1 } else { chunk };
let num_threads = match runtime.current_parallel_region() {
Some(region) => region.num_threads,
None => 1,
};
let tid = (gtid % num_threads as i32).max(0) as u32;
match schedule::base_schedule(sched) {
schedule::SCHEDULE_STATIC => {
let total_iters = (upper - lower) / incr.abs().max(1) + 1;
let (thread_lower, thread_upper, is_last) = compute_static_bounds(
total_iters,
effective_chunk,
num_threads,
tid,
lower,
incr,
upper,
);
*plower = thread_lower.max(0) as u64;
*pupper = thread_upper.max(0) as u64;
*pstride = incr;
*plastiter = if is_last { 1 } else { 0 };
}
_ => {
let total_iters = (upper - lower) / incr.abs().max(1) + 1;
if total_iters > 0 {
let chunk_size = effective_chunk * incr;
let thread_lower = lower + tid as i64 * chunk_size;
let thread_upper = (thread_lower + chunk_size - incr).min(upper);
*plower = thread_lower.max(0) as u64;
*pupper = thread_upper.max(0) as u64;
*pstride = incr;
*plastiter = 0;
} else {
*plower = 0;
*pupper = 0;
*pstride = incr;
*plastiter = 0;
}
}
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_fini(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.barrier.wait();
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_dynamic_init_4(
_loc: *const Ident,
_gtid: i32,
sched: i32,
lower: i32,
upper: i32,
stride: i32,
chunk: i32,
) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
ws.lower = lower as i64;
ws.upper = upper as i64;
ws.stride = stride as i64;
ws.incr = stride as i64;
ws.chunk = if chunk > 0 { chunk as i64 } else { 1 };
ws.schedule = sched;
ws.next_iter = lower as i64;
ws.num_threads = region.num_threads;
if schedule::base_schedule(sched) == schedule::SCHEDULE_GUIDED {
ws.iterations_remaining = if stride != 0 {
((upper as i64 - lower as i64) / stride as i64).abs() + 1
} else {
0
};
} else {
ws.iterations_remaining = 0;
}
ws.initialized = true;
region.workshare.active.store(true, Ordering::Release);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_dynamic_init_8(
_loc: *const Ident,
_gtid: i32,
sched: i32,
lower: i64,
upper: i64,
stride: i64,
chunk: i64,
) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
ws.lower = lower;
ws.upper = upper;
ws.stride = stride;
ws.incr = stride;
ws.chunk = if chunk > 0 { chunk } else { 1 };
ws.schedule = sched;
ws.next_iter = lower;
ws.num_threads = region.num_threads;
if schedule::base_schedule(sched) == schedule::SCHEDULE_GUIDED {
ws.iterations_remaining = if stride != 0 {
((upper - lower) / stride).abs() + 1
} else {
0
};
} else {
ws.iterations_remaining = 0;
}
ws.initialized = true;
region.workshare.active.store(true, Ordering::Release);
}
}
fn guided_chunk_size(remaining: i64, num_threads: u32) -> i64 {
let base = remaining / num_threads as i64;
base.max(OMP_GUIDED_MIN_CHUNK)
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_next_4(
_loc: *const Ident,
_gtid: i32,
plastiter: *mut i32,
plower: *mut i32,
pupper: *mut i32,
pstride: *mut i32,
) -> i32 { unsafe {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
if !ws.initialized {
*plower = 0;
*pupper = 0;
*pstride = ws.stride as i32;
*plastiter = 0;
return 0;
}
let current = ws.next_iter;
if current > ws.upper {
*plower = 0;
*pupper = 0;
*pstride = ws.stride as i32;
*plastiter = 0;
return 0;
}
let base_sched = schedule::base_schedule(ws.schedule);
let chunk_len = match base_sched {
schedule::SCHEDULE_DYNAMIC => ws.chunk,
schedule::SCHEDULE_GUIDED => guided_chunk_size(ws.iterations_remaining, ws.num_threads),
_ => ws.chunk,
};
let chunk_end = (current + (chunk_len - 1) * ws.stride).min(ws.upper);
ws.next_iter = chunk_end + ws.stride;
if base_sched == schedule::SCHEDULE_GUIDED {
ws.iterations_remaining -= chunk_len;
if ws.iterations_remaining < 0 {
ws.iterations_remaining = 0;
}
}
*plower = current as i32;
*pupper = chunk_end as i32;
*pstride = ws.stride as i32;
*plastiter = if chunk_end >= ws.upper { 1 } else { 0 };
return 1;
}
0
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_next_8(
_loc: *const Ident,
_gtid: i32,
plastiter: *mut i32,
plower: *mut i64,
pupper: *mut i64,
pstride: *mut i64,
) -> i32 { unsafe {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
if !ws.initialized {
*plower = 0;
*pupper = 0;
*pstride = ws.stride;
*plastiter = 0;
return 0;
}
let current = ws.next_iter;
if current > ws.upper {
*plower = 0;
*pupper = 0;
*pstride = ws.stride;
*plastiter = 0;
return 0;
}
let base_sched = schedule::base_schedule(ws.schedule);
let chunk_len = match base_sched {
schedule::SCHEDULE_DYNAMIC => ws.chunk,
schedule::SCHEDULE_GUIDED => guided_chunk_size(ws.iterations_remaining, ws.num_threads),
_ => ws.chunk,
};
let chunk_end = (current + (chunk_len - 1) * ws.stride).min(ws.upper);
ws.next_iter = chunk_end + ws.stride;
if base_sched == schedule::SCHEDULE_GUIDED {
ws.iterations_remaining -= chunk_len;
if ws.iterations_remaining < 0 {
ws.iterations_remaining = 0;
}
}
*plower = current;
*pupper = chunk_end;
*pstride = ws.stride;
*plastiter = if chunk_end >= ws.upper { 1 } else { 0 };
return 1;
}
0
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_fini_4(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.workshare.active.store(false, Ordering::Release);
region.barrier.wait();
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_fini_8(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.workshare.active.store(false, Ordering::Release);
region.barrier.wait();
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dist_for_static_init_4(
_loc: *const Ident,
_gtid: i32,
_sched: i32,
plastiter: *mut i32,
plower: *mut i32,
pupper: *mut i32,
pupper_dist: *mut i32,
stride: i32,
chunk: i32,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower as i64;
let upper = *pupper as i64;
let effective_chunk = if chunk == 0 { 1i64 } else { chunk as i64 };
let stride_i64 = stride as i64;
let total_iters = if stride_i64 != 0 {
((upper - lower) / stride_i64).abs() + 1
} else {
0
};
let num_chunks = (total_iters + effective_chunk - 1) / effective_chunk;
let num_teams = match runtime.current_parallel_region() {
Some(region) => region.num_threads as i64,
None => 1,
};
let tid = 0i64;
let chunks_per_team = num_chunks / num_teams;
let remainder = num_chunks % num_teams;
let (start_chunk, end_chunk) = if tid < remainder {
(
tid * (chunks_per_team + 1),
(tid + 1) * (chunks_per_team + 1) - 1,
)
} else {
(
remainder * (chunks_per_team + 1) + (tid - remainder) * chunks_per_team,
remainder * (chunks_per_team + 1) + (tid - remainder + 1) * chunks_per_team - 1,
)
};
if end_chunk >= start_chunk && start_chunk < num_chunks {
let team_lower = lower + start_chunk * effective_chunk * stride_i64;
let team_upper =
(lower + end_chunk * effective_chunk * stride_i64 + (effective_chunk - 1) * stride_i64)
.min(upper);
*plower = team_lower as i32;
*pupper = team_upper as i32;
*pupper_dist = team_upper as i32;
*plastiter = if end_chunk == num_chunks - 1 { 1 } else { 0 };
} else {
*plower = 0;
*pupper = 0;
*pupper_dist = 0;
*plastiter = 0;
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dist_for_static_init_8(
_loc: *const Ident,
_gtid: i32,
_sched: i32,
plastiter: *mut i32,
plower: *mut i64,
pupper: *mut i64,
pupper_dist: *mut i64,
stride: i64,
chunk: i64,
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let lower = *plower;
let upper = *pupper;
let effective_chunk = if chunk == 0 { 1 } else { chunk };
let total_iters = if stride != 0 {
((upper - lower) / stride).abs() + 1
} else {
0
};
let num_chunks = (total_iters + effective_chunk - 1) / effective_chunk;
let num_teams = match runtime.current_parallel_region() {
Some(region) => region.num_threads as i64,
None => 1,
};
let tid = 0i64;
let chunks_per_team = num_chunks / num_teams;
let remainder = num_chunks % num_teams;
let (start_chunk, end_chunk) = if tid < remainder {
(
tid * (chunks_per_team + 1),
(tid + 1) * (chunks_per_team + 1) - 1,
)
} else {
(
remainder * (chunks_per_team + 1) + (tid - remainder) * chunks_per_team,
remainder * (chunks_per_team + 1) + (tid - remainder + 1) * chunks_per_team - 1,
)
};
if end_chunk >= start_chunk && start_chunk < num_chunks {
let team_lower = lower + start_chunk * effective_chunk * stride;
let team_upper =
(lower + end_chunk * effective_chunk * stride + (effective_chunk - 1) * stride)
.min(upper);
*plower = team_lower;
*pupper = team_upper;
*pupper_dist = team_upper;
*plastiter = if end_chunk == num_chunks - 1 { 1 } else { 0 };
} else {
*plower = 0;
*pupper = 0;
*pupper_dist = 0;
*plastiter = 0;
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_taskloop(
_loc: *const Ident,
gtid: i32,
task_func: unsafe extern "C" fn(i32, *mut TaskDescriptor),
lower: i64,
upper: i64,
stride: i64,
grainsize: i64,
num_tasks: i64,
flags: i32,
) {
let runtime = &*GLOBAL_RUNTIME;
let state = TaskLoopState::new(lower, upper, stride, grainsize, num_tasks);
runtime.debug_log(
1,
&format!(
"taskloop: lower={}, upper={}, stride={}, grainsize={}, num_tasks={}",
lower, upper, stride, grainsize, num_tasks
),
);
while let Some((chunk_lower, chunk_upper)) = state.next_chunk() {
let task_id = runtime.task_queue.allocate_task_id();
let mut task = TaskDescriptor::new(task_id, gtid);
task.task_fn = Some(task_func);
task.flags = flags;
let task_arc = Arc::new(task);
runtime.task_queue.add_to_taskgroup(Arc::clone(&task_arc));
runtime.task_queue.enqueue_task(task_arc);
}
runtime.task_queue.taskwait(gtid);
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_sections_init(
_loc: *const Ident,
_gtid: i32,
num_sections: u32,
) -> u32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
ws.lower = 0;
ws.upper = num_sections as i64;
ws.next_iter = 0;
ws.initialized = true;
}
num_sections
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_next_section(_loc: *const Ident, _gtid: i32) -> u32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let mut ws = region.workshare.state.lock().unwrap();
if !ws.initialized {
return u32::MAX;
}
let current = ws.next_iter;
if current < ws.upper {
ws.next_iter = current + 1;
return current as u32;
}
}
u32::MAX
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_ordered(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
while region
.ordered
.in_ordered
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
std::hint::spin_loop();
}
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_end_ordered(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.ordered.in_ordered.store(false, Ordering::Release);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_barrier(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.barrier.set_pattern(BARRIER_PATTERN_PARALLEL);
region.barrier.wait();
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_cancel_barrier(_loc: *const Ident, _gtid: i32) -> i32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
if region.barrier.wait_cancellable() {
return 1;
}
}
0
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_master(_loc: *const Ident, gtid: i32) -> i32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
if region.master.is_master(gtid) {
return 1;
}
}
0
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_end_master(_loc: *const Ident, _gtid: i32) {
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_single(_loc: *const Ident, _gtid: i32) -> i32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
if region.single.try_acquire() {
return 1;
}
}
0
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_end_single(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
region.barrier.set_pattern(BARRIER_PATTERN_SINGLE);
region.barrier.wait();
if region.master.is_master(_gtid) {
region.single.reset();
}
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_critical(_loc: *const Ident, _gtid: i32, lock_ptr: *mut ()) {
let runtime = &*GLOBAL_RUNTIME;
let key = if lock_ptr.is_null() {
0usize
} else {
lock_ptr as usize
};
let name = format!("crit_{:x}", key);
let lock = runtime.critical_locks.get_or_create(&name);
let guard = lock.lock().unwrap();
let guard_raw = Box::into_raw(Box::new(guard));
CRITICAL_GUARDS.with(|guards| {
guards.borrow_mut().insert(key, guard_raw as usize);
});
}
std::thread_local! {
static CRITICAL_GUARDS: std::cell::RefCell<HashMap<usize, usize>> =
std::cell::RefCell::new(HashMap::new());
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_end_critical(_loc: *const Ident, _gtid: i32, lock_ptr: *mut ()) { unsafe {
let key = if lock_ptr.is_null() {
0usize
} else {
lock_ptr as usize
};
CRITICAL_GUARDS.with(|guards| {
let mut guards = guards.borrow_mut();
if let Some(guard_addr) = guards.remove(&key) {
let guard: Box<std::sync::MutexGuard<'static, ()>> =
Box::from_raw(guard_addr as *mut std::sync::MutexGuard<'static, ()>);
drop(guard);
}
});
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_flush(_loc: *const Ident) {
std::sync::atomic::fence(Ordering::SeqCst);
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_global_thread_num(_loc: *const Ident) -> i32 {
THREAD_GLOBAL_ID.with(|id| id.get())
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_bound_thread_num(_loc: *const Ident) -> i32 {
THREAD_GLOBAL_ID.with(|id| id.get())
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task_alloc(
_loc: *const Ident,
gtid: i32,
flags: i32,
_size_of_task: usize,
size_of_shareds: usize,
task_func: unsafe extern "C" fn(i32, *mut TaskDescriptor),
) -> *mut TaskDescriptor { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let task_id = runtime.task_queue.allocate_task_id();
let mut task = TaskDescriptor::new(task_id, gtid);
task.flags = flags;
task.task_fn = Some(task_func);
if size_of_shareds > 0 {
let layout = std::alloc::Layout::from_size_align(size_of_shareds.max(1), 8).unwrap();
task.shareds = std::alloc::alloc(layout);
task.shareds_size = size_of_shareds;
std::ptr::write_bytes(task.shareds, 0, size_of_shareds);
}
Box::into_raw(Box::new(task))
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task(
_loc: *const Ident,
_gtid: i32,
task_ptr: *mut TaskDescriptor,
) -> i32 { unsafe {
if task_ptr.is_null() {
return 0;
}
let runtime = &*GLOBAL_RUNTIME;
let task = Box::from_raw(task_ptr);
let task_arc = Arc::new(*task);
runtime.task_queue.add_to_taskgroup(Arc::clone(&task_arc));
runtime.task_queue.enqueue_task(task_arc);
0
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task_with_deps(
_loc: *const Ident,
_gtid: i32,
task_ptr: *mut TaskDescriptor,
dep_count: i32,
_dep_list: *const u8,
_no_of_deps_all: i32,
_dep_info: *const u8,
) -> i32 { unsafe {
if task_ptr.is_null() {
return 0;
}
let runtime = &*GLOBAL_RUNTIME;
let mut task = Box::from_raw(task_ptr);
task.dep_count = dep_count as u32;
task.deps_satisfied = true;
let task_arc = Arc::new(*task);
runtime.task_queue.add_to_taskgroup(Arc::clone(&task_arc));
runtime.task_queue.enqueue_task(task_arc);
0
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task_begin_if0(
_loc: *const Ident,
_gtid: i32,
task_ptr: *mut TaskDescriptor,
) { unsafe {
if task_ptr.is_null() {
return;
}
let task = &mut *task_ptr;
task.undeferred = true;
task.status.store(1, Ordering::Release);
if let Some(func) = task.task_fn {
func(_gtid, task_ptr);
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task_complete_if0(
_loc: *const Ident,
_gtid: i32,
task_ptr: *mut TaskDescriptor,
) { unsafe {
if task_ptr.is_null() {
return;
}
let task = &mut *task_ptr;
task.status.store(2, Ordering::Release);
if !task.shareds.is_null() && task.shareds_size > 0 {
let layout = std::alloc::Layout::from_size_align(task.shareds_size, 8).unwrap();
std::alloc::dealloc(task.shareds, layout);
task.shareds = std::ptr::null_mut();
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_taskwait(_loc: *const Ident, gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
runtime.task_queue.taskwait(gtid);
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_taskyield(_loc: *const Ident, _gtid: i32) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
if let Some(task) = runtime.task_queue.steal_task() {
if let Some(f) = task.task_fn {
let gtid = runtime.get_gtid();
task.status.store(1, Ordering::Release);
f(gtid, Arc::as_ptr(&task) as *mut TaskDescriptor);
task.status.store(2, Ordering::Release);
}
} else {
std::thread::yield_now();
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_taskgroup(_loc: *const Ident, _gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
runtime.task_queue.begin_taskgroup();
THREAD_TASK_STATE.with(|ts| {
ts.borrow_mut().taskgroup_depth += 1;
ts.borrow_mut().in_taskgroup = true;
});
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_taskgroup_end(_loc: *const Ident, gtid: i32) {
let runtime = &*GLOBAL_RUNTIME;
runtime.task_queue.end_taskgroup(gtid);
THREAD_TASK_STATE.with(|ts| {
let mut ts = ts.borrow_mut();
if ts.taskgroup_depth > 0 {
ts.taskgroup_depth -= 1;
}
if ts.taskgroup_depth == 0 {
ts.in_taskgroup = false;
}
});
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_target_task_alloc(
_loc: *const Ident,
gtid: i32,
flags: i32,
size_of_task: usize,
size_of_shareds: usize,
task_func: unsafe extern "C" fn(i32, *mut TaskDescriptor),
) -> *mut TaskDescriptor { unsafe {
__kmpc_omp_task_alloc(
_loc,
gtid,
flags | TASK_FLAG_TARGET,
size_of_task,
size_of_shareds,
task_func,
)
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_simd(_loc: *const Ident) {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum X86SimdWidth {
SSE = 4,
AVX = 8,
AVX512 = 16,
}
impl X86SimdWidth {
pub fn f32_count(&self) -> usize {
match self {
X86SimdWidth::SSE => 4,
X86SimdWidth::AVX => 8,
X86SimdWidth::AVX512 => 16,
}
}
pub fn f64_count(&self) -> usize {
match self {
X86SimdWidth::SSE => 2,
X86SimdWidth::AVX => 4,
X86SimdWidth::AVX512 => 8,
}
}
pub fn i32_count(&self) -> usize {
self.f32_count()
}
pub fn i64_count(&self) -> usize {
self.f64_count()
}
pub fn i8_count(&self) -> usize {
self.byte_width()
}
pub fn i16_count(&self) -> usize {
self.byte_width() / 2
}
pub fn byte_width(&self) -> usize {
match self {
X86SimdWidth::SSE => 16,
X86SimdWidth::AVX => 32,
X86SimdWidth::AVX512 => 64,
}
}
pub fn bit_width(&self) -> usize {
self.byte_width() * 8
}
pub fn name(&self) -> &'static str {
match self {
X86SimdWidth::SSE => "SSE (128-bit)",
X86SimdWidth::AVX => "AVX (256-bit)",
X86SimdWidth::AVX512 => "AVX-512 (512-bit)",
}
}
}
pub fn x86_simd_vector_length() -> X86SimdWidth {
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
{
if is_x86_feature_detected!("avx512f") {
return X86SimdWidth::AVX512;
}
if is_x86_feature_detected!("avx") || is_x86_feature_detected!("avx2") {
return X86SimdWidth::AVX;
}
X86SimdWidth::SSE
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
{
X86SimdWidth::SSE
}
}
pub fn kmpc_simd_width_i32() -> i32 {
x86_simd_vector_length().i32_count() as i32
}
pub fn kmpc_simd_width_i64() -> i32 {
x86_simd_vector_length().i64_count() as i32
}
pub fn kmpc_simd_width_f32() -> i32 {
x86_simd_vector_length().f32_count() as i32
}
pub fn kmpc_simd_width_f64() -> i32 {
x86_simd_vector_length().f64_count() as i32
}
pub fn kmpc_simd_width_i8() -> i32 {
x86_simd_vector_length().i8_count() as i32
}
pub fn kmpc_simd_width_i16() -> i32 {
x86_simd_vector_length().i16_count() as i32
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_reduce(
_loc: *const Ident,
_gtid: i32,
_num_vars: i32,
_size: usize,
_data: *mut u8,
_reduce_func: unsafe extern "C" fn(*mut u8, *const u8),
_lock: *mut (),
) -> i32 {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let key = _lock as usize;
let ctx = region.get_reduction(key);
if !ctx.active.load(Ordering::Acquire) {
if ctx
.active
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
return 1; }
}
let _guard = ctx.lock.lock().unwrap();
return 0;
}
0
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_end_reduce(_loc: *const Ident, _gtid: i32, _lock: *mut ()) {
let runtime = &*GLOBAL_RUNTIME;
if let Some(region) = runtime.current_parallel_region() {
let key = _lock as usize;
let ctx = region.get_reduction(key);
ctx.active.store(false, Ordering::Release);
region.barrier.set_pattern(BARRIER_PATTERN_REDUCTION);
region.barrier.wait();
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_reduce_nowait(
_loc: *const Ident,
_gtid: i32,
_num_vars: i32,
_size: usize,
_data: *mut u8,
_reduce_func: unsafe extern "C" fn(*mut u8, *const u8),
_lock: *mut (),
) -> i32 { unsafe {
__kmpc_reduce(_loc, _gtid, _num_vars, _size, _data, _reduce_func, _lock)
}}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_num_threads() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
match runtime.current_parallel_region() {
Some(region) => region.num_threads as i32,
None => 1,
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_thread_num() -> i32 {
THREAD_GLOBAL_ID.with(|id| {
let gtid = id.get();
let runtime = &*GLOBAL_RUNTIME;
match runtime.current_parallel_region() {
Some(region) => gtid % region.num_threads as i32,
None => 0,
}
})
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_max_threads() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
runtime.get_num_threads() as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_num_threads(num_threads: i32) {
if num_threads > 0 {
THREAD_ICV.with(|icv| {
icv.borrow_mut().num_threads = num_threads as u32;
});
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_num_procs() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
runtime.num_procs as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_in_parallel() -> i32 {
THREAD_ICV.with(|icv| if icv.borrow().in_parallel { 1 } else { 0 })
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_dynamic() -> i32 {
THREAD_ICV.with(|icv| if icv.borrow().dynamic { 1 } else { 0 })
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_dynamic(dynamic: i32) {
THREAD_ICV.with(|icv| {
icv.borrow_mut().dynamic = dynamic != 0;
});
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_nested() -> i32 {
THREAD_ICV.with(|icv| if icv.borrow().nested { 1 } else { 0 })
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_nested(nested: i32) {
THREAD_ICV.with(|icv| {
icv.borrow_mut().nested = nested != 0;
});
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_schedule(kind: *mut i32, modifier: *mut i32) {
THREAD_ICV.with(|icv| {
let icv = icv.borrow();
if !kind.is_null() {
unsafe { *kind = icv.schedule_kind };
}
if !modifier.is_null() {
unsafe { *modifier = icv.schedule_chunk as i32 };
}
});
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_schedule(kind: i32, modifier: i32) {
THREAD_ICV.with(|icv| {
let mut icv = icv.borrow_mut();
icv.schedule_kind = kind;
icv.schedule_chunk = modifier as i64;
});
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_thread_limit() -> i32 {
THREAD_ICV.with(|icv| icv.borrow().thread_limit as i32)
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_max_active_levels() -> i32 {
THREAD_ICV.with(|icv| icv.borrow().max_active_levels as i32)
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_max_active_levels(max_levels: i32) {
THREAD_ICV.with(|icv| {
icv.borrow_mut().max_active_levels = max_levels.max(0) as u32;
});
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_level() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
runtime.get_active_level() as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_ancestor_thread_num(level: i32) -> i32 {
if level < 0 {
return -1;
}
let runtime = &*GLOBAL_RUNTIME;
runtime.get_ancestor_thread_num(level as u32)
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_team_size(level: i32) -> i32 {
if level < 0 {
return -1;
}
let runtime = &*GLOBAL_RUNTIME;
runtime.get_team_size(level as u32)
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_active_level() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
runtime.get_active_level() as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_wtime() -> f64 {
let runtime = &*GLOBAL_RUNTIME;
runtime.start_time.elapsed().as_secs_f64()
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_wtick() -> f64 {
1e-9
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_init_lock(lock: *mut *mut OMPLock) {
if lock.is_null() {
return;
}
unsafe {
*lock = Box::into_raw(Box::new(OMPLock::new()));
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_destroy_lock(lock: *mut *mut OMPLock) {
if lock.is_null() {
return;
}
unsafe {
if !(*lock).is_null() {
let _ = Box::from_raw(*lock);
*lock = std::ptr::null_mut();
}
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_lock(lock: *mut *mut OMPLock) {
if lock.is_null() {
return;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return;
}
&**lock
};
let gtid = THREAD_GLOBAL_ID.with(|id| id.get());
lock_ref.set(gtid);
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_unset_lock(lock: *mut *mut OMPLock) {
if lock.is_null() {
return;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return;
}
&**lock
};
lock_ref.unset();
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_test_lock(lock: *mut *mut OMPLock) -> i32 {
if lock.is_null() {
return 0;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return 0;
}
&**lock
};
let gtid = THREAD_GLOBAL_ID.with(|id| id.get());
if lock_ref.test(gtid) {
1
} else {
0
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_init_nest_lock(lock: *mut *mut OMPNestLock) {
if lock.is_null() {
return;
}
unsafe {
*lock = Box::into_raw(Box::new(OMPNestLock::new()));
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_destroy_nest_lock(lock: *mut *mut OMPNestLock) {
if lock.is_null() {
return;
}
unsafe {
if !(*lock).is_null() {
let _ = Box::from_raw(*lock);
*lock = std::ptr::null_mut();
}
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_nest_lock(lock: *mut *mut OMPNestLock) {
if lock.is_null() {
return;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return;
}
&**lock
};
let gtid = THREAD_GLOBAL_ID.with(|id| id.get());
lock_ref.set(gtid);
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_unset_nest_lock(lock: *mut *mut OMPNestLock) {
if lock.is_null() {
return;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return;
}
&**lock
};
lock_ref.unset();
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_test_nest_lock(lock: *mut *mut OMPNestLock) -> i32 {
if lock.is_null() {
return 0;
}
let lock_ref = unsafe {
if (*lock).is_null() {
return 0;
}
&**lock
};
let gtid = THREAD_GLOBAL_ID.with(|id| id.get());
if lock_ref.test(gtid) {
lock_ref.depth() as i32
} else {
0
}
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_register_lib(_desc: *const u8) {}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_unregister_lib(_desc: *const u8) {}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target(
_device_id: i64,
_target_id: *const u8,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
) -> i32 {
-1 }
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_nowait(
_device_id: i64,
_target_id: *const u8,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
) -> i32 {
-1
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_teams(
_device_id: i64,
_target_id: *const u8,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
_num_teams: i32,
_thread_limit: i32,
) -> i32 {
-1
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_teams_nowait(
_device_id: i64,
_target_id: *const u8,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
_num_teams: i32,
_thread_limit: i32,
) -> i32 {
-1
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_data_begin(
_device_id: i64,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
) {
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_data_end(
_device_id: i64,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
) {
}
#[unsafe(no_mangle)]
pub extern "C" fn __tgt_target_data_update(
_device_id: i64,
_arg_num: i32,
_args_base: *mut *mut u8,
_args: *mut *mut u8,
_arg_sizes: *mut i64,
_arg_types: *mut i64,
) {
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_alloc(size: usize, _device_num: i32) -> *mut u8 {
if size == 0 {
return std::ptr::null_mut();
}
let layout = std::alloc::Layout::from_size_align(size, 8).unwrap();
unsafe { std::alloc::alloc(layout) }
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_free(ptr: *mut u8, _device_num: i32) {
let _ = ptr;
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_memcpy(
dst: *mut u8,
src: *const u8,
length: usize,
_dst_offset: usize,
_src_offset: usize,
_dst_device_num: i32,
_src_device_num: i32,
) -> i32 {
if dst.is_null() || src.is_null() || length == 0 {
return -1;
}
unsafe { std::ptr::copy_nonoverlapping(src, dst, length) };
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_memcpy_rect(
dst: *mut u8,
src: *const u8,
_element_size: usize,
_num_dims: i32,
_volume: *const usize,
_dst_offsets: *const usize,
_src_offsets: *const usize,
_dst_dimensions: *const usize,
_src_dimensions: *const usize,
_dst_device_num: i32,
_src_device_num: i32,
) -> i32 {
if dst.is_null() || src.is_null() {
return -1;
}
let total_size: usize = unsafe {
if _num_dims > 0 && !_volume.is_null() {
*_volume
} else {
0
}
};
if total_size > 0 {
unsafe { std::ptr::copy_nonoverlapping(src, dst, total_size) };
}
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_associate_ptr(
_host_ptr: *const u8,
_device_ptr: *const u8,
_size: usize,
_device_offset: usize,
_device_num: i32,
) -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_target_disassociate_ptr(_host_ptr: *const u8, _device_num: i32) -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_is_initial_device() -> i32 {
1
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_initial_device() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_num_devices() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_default_device() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_default_device(_device_num: i32) {}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_part_num_threads() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
match runtime.current_parallel_region() {
Some(region) => region.num_threads as i32,
None => 1,
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_cancellation() -> i32 {
THREAD_ICV.with(|icv| if icv.borrow().cancellation { 1 } else { 0 })
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_proc_bind() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
*runtime.proc_bind.read().unwrap() as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_num_places() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
runtime.places.read().unwrap().len() as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_place_num_procs(place_num: i32) -> i32 {
let runtime = &*GLOBAL_RUNTIME;
let places = runtime.places.read().unwrap();
if place_num >= 0 && (place_num as usize) < places.len() {
places[place_num as usize].len() as i32
} else {
0
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_place_proc_ids(place_num: i32, ids: *mut i32) {
if ids.is_null() {
return;
}
let runtime = &*GLOBAL_RUNTIME;
let places = runtime.places.read().unwrap();
if place_num >= 0 && (place_num as usize) < places.len() {
let place = &places[place_num as usize];
for (i, &proc_id) in place.iter().enumerate() {
unsafe {
*ids.add(i) = proc_id;
}
}
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_place_num() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_partition_num_places() -> i32 {
1
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_partition_place_nums(place_nums: *mut i32) {
if !place_nums.is_null() {
unsafe {
*place_nums = 0;
}
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_team_num() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_num_teams() -> i32 {
1
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_num_teams(_num_teams: i32) {}
#[unsafe(no_mangle)]
pub extern "C" fn omp_set_teams_thread_limit(_thread_limit: i32) {}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_max_task_priority() -> i32 {
1
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_pause_resource(_kind: i32, _device_num: i32) -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_pause_resource_all(_kind: i32) -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_display_env(_verbose: i32) {
eprintln!("OPENMP DISPLAY ENVIRONMENT");
eprintln!(" OMP_NUM_THREADS = {}", omp_get_max_threads());
eprintln!(
" OMP_DYNAMIC = {}",
if omp_get_dynamic() != 0 {
"TRUE"
} else {
"FALSE"
}
);
eprintln!(
" OMP_NESTED = {}",
if omp_get_nested() != 0 {
"TRUE"
} else {
"FALSE"
}
);
eprintln!(" OMP_MAX_ACTIVE_LEVELS = {}", omp_get_max_active_levels());
}
#[unsafe(no_mangle)]
pub extern "C" fn __kmpc_begin(_loc: *const Ident, _flags: i32) {
let runtime = &*GLOBAL_RUNTIME;
runtime.initialize();
}
#[unsafe(no_mangle)]
pub extern "C" fn __kmpc_end(_loc: *const Ident) {}
pub fn omp_initialize() {
let runtime = &*GLOBAL_RUNTIME;
runtime.initialize();
}
pub fn omp_finalize() {
let runtime = &*GLOBAL_RUNTIME;
runtime.shutdown();
}
pub fn default_ident() -> Ident {
Ident::default()
}
pub fn get_runtime() -> &'static Arc<X86OpenMP> {
&GLOBAL_RUNTIME
}
pub fn current_gtid() -> i32 {
THREAD_GLOBAL_ID.with(|id| id.get())
}
pub fn with_current_icv<F, R>(f: F) -> R
where
F: FnOnce(&mut ICVStorage) -> R,
{
THREAD_ICV.with(|icv| f(&mut icv.borrow_mut()))
}
#[cfg(test)]
mod tests {
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(dead_code)]
use super::*;
use std::sync::atomic::Ordering;
fn init_test() {
THREAD_GLOBAL_ID.with(|id| id.set(0));
THREAD_ICV.with(|icv| {
let mut i = icv.borrow_mut();
*i = ICVStorage::default();
i.num_threads = 4;
i.thread_limit = 256;
});
}
fn with_region<F>(runtime: &X86OpenMP, num_threads: u32, f: F)
where
F: FnOnce(),
{
let region = Arc::new(ParallelRegionState::new(0, num_threads));
runtime.push_parallel_region(region);
f();
runtime.pop_parallel_region();
}
#[test]
fn test_icv_defaults() {
let icv = ICVStorage::default();
assert_eq!(icv.num_threads, 0);
assert_eq!(icv.thread_limit, OMP_MAX_THREADS_X86);
assert!(!icv.dynamic);
assert!(!icv.nested);
assert_eq!(icv.schedule_kind, schedule::SCHEDULE_STATIC);
}
#[test]
fn test_icv_inherit_parallel_flag() {
let parent = ICVStorage::default();
let child = parent.inherit();
assert!(child.in_parallel);
assert_eq!(child.active_level, parent.active_level + 1);
}
#[test]
fn test_icv_effective_num_threads() {
let mut icv = ICVStorage::default();
icv.num_threads = 8;
icv.thread_limit = 4;
assert_eq!(icv.effective_num_threads(16), 4);
icv.num_threads = 0;
assert_eq!(icv.effective_num_threads(16), 16);
icv.dynamic = true;
assert!(icv.effective_num_threads(4) <= 4);
}
#[test]
fn test_icv_effective_schedule() {
let icv = ICVStorage {
schedule_kind: schedule::SCHEDULE_RUNTIME,
default_schedule: schedule::SCHEDULE_GUIDED,
default_chunk: 7,
..Default::default()
};
let (kind, chunk) = icv.effective_schedule();
assert_eq!(kind, schedule::SCHEDULE_GUIDED);
assert_eq!(chunk, 7);
}
#[test]
fn test_parse_schedule_env() {
assert_eq!(
parse_schedule_env("static,4"),
(schedule::SCHEDULE_STATIC, 4)
);
assert_eq!(
parse_schedule_env("dynamic"),
(schedule::SCHEDULE_DYNAMIC, 0)
);
assert_eq!(
parse_schedule_env("guided,10"),
(schedule::SCHEDULE_GUIDED, 10)
);
assert_eq!(parse_schedule_env("auto"), (schedule::SCHEDULE_AUTO, 0));
}
#[test]
fn test_parse_size() {
assert_eq!(parse_size("1024"), Some(1024));
assert_eq!(parse_size("1K"), Some(1024));
assert_eq!(parse_size("1M"), Some(1024 * 1024));
assert_eq!(parse_size("2G"), Some(2 * 1024 * 1024 * 1024));
assert_eq!(parse_size("abc"), None);
}
#[test]
fn test_omp_parse_environment_no_crash() {
let mut icv = ICVStorage::default();
omp_parse_environment(&mut icv);
}
#[test]
fn test_schedule_base() {
assert_eq!(
schedule::base_schedule(schedule::SCHEDULE_STATIC),
schedule::SCHEDULE_STATIC
);
assert_eq!(
schedule::base_schedule(schedule::SCHEDULE_STATIC | schedule::SCHEDULE_MONOTONIC),
schedule::SCHEDULE_STATIC
);
assert!(schedule::is_monotonic(
schedule::SCHEDULE_STATIC | schedule::SCHEDULE_MONOTONIC
));
assert!(!schedule::is_monotonic(schedule::SCHEDULE_STATIC));
assert!(schedule::is_nonmonotonic(
schedule::SCHEDULE_STATIC | schedule::SCHEDULE_NONMONOTONIC
));
}
#[test]
fn test_omp_lock_basic() {
let lock = OMPLock::new();
assert!(!lock.is_locked());
lock.set(42);
assert!(lock.is_locked());
assert_eq!(lock.owner(), 42);
lock.unset();
assert!(!lock.is_locked());
}
#[test]
fn test_omp_lock_test() {
let lock = OMPLock::new();
assert!(lock.test(1));
assert!(!lock.test(2));
lock.unset();
assert!(lock.test(2));
}
#[test]
fn test_omp_lock_contention() {
let lock = Arc::new(OMPLock::new());
let c = Arc::clone(&lock);
let h = std::thread::spawn(move || {
c.set(1);
std::thread::sleep(Duration::from_millis(10));
c.unset();
});
std::thread::sleep(Duration::from_millis(5));
lock.set(0);
lock.unset();
h.join().unwrap();
}
#[test]
fn test_nest_lock_basic() {
let lock = OMPNestLock::new();
lock.set(1);
lock.set(1); assert!(lock.is_locked());
lock.unset();
assert!(lock.is_locked());
lock.unset();
assert!(!lock.is_locked());
}
#[test]
fn test_nest_lock_test() {
let lock = OMPNestLock::new();
assert!(lock.test(1));
assert!(lock.test(1));
assert!(!lock.test(2));
}
#[test]
fn test_barrier_basic() {
let b = Arc::new(TeamBarrier::new(2));
let b2 = Arc::clone(&b);
let h = std::thread::spawn(move || b2.wait());
b.wait();
h.join().unwrap();
}
#[test]
fn test_barrier_cancellable() {
let b = Arc::new(TeamBarrier::new(3));
let b1 = Arc::clone(&b);
let b2 = Arc::clone(&b);
let h = std::thread::spawn(move || b1.wait_cancellable());
std::thread::sleep(Duration::from_millis(10));
b2.cancel();
assert!(h.join().unwrap());
}
#[test]
fn test_barrier_reset() {
let b = TeamBarrier::new(2);
b.wait();
b.wait();
b.reset(3);
assert_eq!(b.count.load(Ordering::Acquire), 3);
}
#[test]
fn test_single_acquire() {
let s = SingleState::new();
assert!(s.try_acquire());
assert!(!s.try_acquire());
s.reset();
assert!(s.try_acquire());
}
#[test]
fn test_master_state() {
let m = MasterState::new(0);
assert!(m.is_master(0));
assert!(!m.is_master(1));
}
#[test]
fn test_ordered_state_default() {
let o = OrderedState::new();
assert!(!o.in_ordered.load(Ordering::Acquire));
}
#[test]
fn test_critical_lock_table_identity() {
let t = CriticalLockTable::new();
let a = t.get_or_create("a");
let b = t.get_or_create("b");
let a2 = t.get_or_create("a");
assert!(Arc::ptr_eq(&a, &a2));
assert!(!Arc::ptr_eq(&a, &b));
}
#[test]
fn test_critical_lock_remove() {
let t = CriticalLockTable::new();
t.get_or_create("x");
assert!(t.remove("x"));
assert!(!t.remove("y"));
}
#[test]
fn test_reduction_table_comprehensive() {
let t = ReductionTable::new();
assert!(t.get("i32_add").is_some());
assert!(t.get("i64_mul").is_some());
assert!(t.get("f32_max").is_some());
assert!(t.get("f64_min").is_some());
assert!(t.get("u32_add").is_some());
assert!(t.get("u64_max").is_some());
assert!(t.get("i8_add").is_some());
assert!(t.get("i16_add").is_some());
assert!(t.get("nonexistent").is_none());
}
#[test]
fn test_reduction_i32_add_fn() {
let t = ReductionTable::new();
let info = t.get("i32_add").unwrap();
let mut v: i32 = -1;
(info.init_fn)(&mut v as *mut i32 as *mut u8);
assert_eq!(v, 0);
let other: i32 = 5;
(info.combine_fn)(
&mut v as *mut i32 as *mut u8,
&other as *const i32 as *const u8,
);
assert_eq!(v, 5);
}
#[test]
fn test_reduction_i32_min() {
let t = ReductionTable::new();
let info = t.get("i32_min").unwrap();
let mut v: i32 = 0;
(info.init_fn)(&mut v as *mut i32 as *mut u8);
assert_eq!(v, i32::MAX);
let s: i32 = -5;
(info.combine_fn)(&mut v as *mut i32 as *mut u8, &s as *const i32 as *const u8);
assert_eq!(v, -5);
}
#[test]
fn test_reduction_f64_mul() {
let t = ReductionTable::new();
let info = t.get("f64_mul").unwrap();
let mut v: f64 = 0.0;
(info.init_fn)(&mut v as *mut f64 as *mut u8);
assert!((v - 1.0).abs() < f64::EPSILON);
let other: f64 = 2.5;
(info.combine_fn)(
&mut v as *mut f64 as *mut u8,
&other as *const f64 as *const u8,
);
assert!((v - 2.5).abs() < f64::EPSILON);
}
#[test]
fn test_reduction_table_register_unregister() {
let mut t = ReductionTable::new();
t.register(
"custom".into(),
ReductionTypeInfo {
type_size: 8,
op: ReductionOp::Custom,
init_fn: |_| {},
combine_fn: |_, _| {},
finalize_fn: Some(|_| {}),
},
);
assert!(t.get("custom").is_some());
assert!(t.unregister("custom").is_some());
assert!(t.get("custom").is_none());
}
#[test]
fn test_reduction_context() {
let ctx = ReductionContext::new();
assert!(!ctx.active.load(Ordering::Acquire));
}
#[test]
fn test_task_descriptor_fields() {
let t = TaskDescriptor::new(42, 7);
assert_eq!(t.task_id, 42);
assert_eq!(t.parent_gtid, 7);
assert!(t.tied);
assert!(!t.is_untied());
assert!(!t.is_final());
}
#[test]
fn test_task_queue_enqueue_dequeue() {
let q = TaskQueue::new();
q.enqueue_task(Arc::new(TaskDescriptor::new(1, 0)));
q.enqueue_task(Arc::new(TaskDescriptor::new(2, 0)));
assert_eq!(q.dequeue_task().unwrap().task_id, 1);
assert_eq!(q.dequeue_task().unwrap().task_id, 2);
assert!(q.dequeue_task().is_none());
}
#[test]
fn test_task_queue_priority() {
let q = TaskQueue::new();
let mut t1 = TaskDescriptor::new(1, 0);
t1.priority = 0;
let mut t2 = TaskDescriptor::new(2, 0);
t2.priority = 10;
let mut t3 = TaskDescriptor::new(3, 0);
t3.priority = 5;
q.enqueue_task(Arc::new(t1));
q.enqueue_task(Arc::new(t2));
q.enqueue_task(Arc::new(t3));
assert_eq!(q.dequeue_task().unwrap().task_id, 2); assert_eq!(q.dequeue_task().unwrap().task_id, 3);
assert_eq!(q.dequeue_task().unwrap().task_id, 1);
}
#[test]
fn test_task_queue_taskgroup() {
let q = TaskQueue::new();
q.begin_taskgroup();
let t = Arc::new(TaskDescriptor::new(1, 0));
t.status.store(2, Ordering::Release);
q.add_to_taskgroup(t);
q.end_taskgroup(0);
}
#[test]
fn test_task_queue_steal() {
let q = TaskQueue::new();
q.enqueue_task(Arc::new(TaskDescriptor::new(1, 0)));
q.enqueue_task(Arc::new(TaskDescriptor::new(2, 0)));
assert_eq!(q.steal_task().unwrap().task_id, 2); assert_eq!(q.steal_task().unwrap().task_id, 1);
}
#[test]
fn test_thread_pool_default() {
let p = ThreadPool::default();
assert_eq!(p.num_threads.load(Ordering::Acquire), 1);
}
#[test]
fn test_thread_pool_wait_policy() {
let p = ThreadPool::new();
p.set_wait_policy(WaitPolicy::Passive);
assert_eq!(*p.wait_policy.read().unwrap(), WaitPolicy::Passive);
}
#[test]
fn test_compute_static_bounds_even() {
let (lo, hi, last) = compute_static_bounds(100, 1, 4, 0, 0, 1, 99);
assert_eq!(lo, 0);
assert_eq!(hi, 24);
assert!(!last);
}
#[test]
fn test_compute_static_bounds_last() {
let (lo, hi, last) = compute_static_bounds(100, 1, 4, 3, 0, 1, 99);
assert_eq!(lo, 75);
assert_eq!(hi, 99);
assert!(last);
}
#[test]
fn test_workshare_state_default() {
let ws = WorkshareState::default();
assert!(!ws.initialized);
assert_eq!(ws.chunk, 1);
}
#[test]
fn test_sections_state_exhaustion() {
let s = SectionsState::new(2);
assert_eq!(s.next(), Some(0));
assert_eq!(s.next(), Some(1));
assert_eq!(s.next(), None);
}
#[test]
fn test_taskloop_state_grainsize() {
let s = TaskLoopState::new(0, 9, 1, 3, 0);
let chunks: Vec<_> = std::iter::from_fn(|| s.next_chunk()).collect();
assert_eq!(chunks.len(), 4);
assert_eq!(chunks[0], (0, 2));
assert_eq!(chunks[3], (9, 9));
}
#[test]
fn test_taskloop_state_num_tasks() {
let s = TaskLoopState::new(0, 9, 1, 0, 5);
let chunks: Vec<_> = std::iter::from_fn(|| s.next_chunk()).collect();
assert_eq!(chunks.len(), 5);
}
#[test]
fn test_taskloop_exhausted() {
let s = TaskLoopState::new(0, 0, 1, 1, 0);
assert!(s.next_chunk().is_some());
assert!(s.next_chunk().is_none());
}
#[test]
fn test_simd_width_values() {
assert_eq!(X86SimdWidth::SSE.f32_count(), 4);
assert_eq!(X86SimdWidth::AVX.f32_count(), 8);
assert_eq!(X86SimdWidth::AVX512.f32_count(), 16);
assert_eq!(X86SimdWidth::SSE.f64_count(), 2);
assert_eq!(X86SimdWidth::AVX.f64_count(), 4);
assert_eq!(X86SimdWidth::AVX512.f64_count(), 8);
}
#[test]
fn test_simd_width_byte() {
assert_eq!(X86SimdWidth::SSE.byte_width(), 16);
assert_eq!(X86SimdWidth::AVX.byte_width(), 32);
assert_eq!(X86SimdWidth::AVX512.byte_width(), 64);
}
#[test]
fn test_simd_vector_length_fallback() {
let w = x86_simd_vector_length();
assert!(w == X86SimdWidth::SSE || w == X86SimdWidth::AVX || w == X86SimdWidth::AVX512);
}
#[test]
fn test_runtime_creation() {
let rt = X86OpenMP::new();
assert!(!rt.is_shutting_down());
assert_eq!(rt.get_active_level(), 0);
}
#[test]
fn test_runtime_initialize_idempotent() {
let rt = X86OpenMP::new();
rt.initialize();
rt.initialize();
assert!(rt.is_initialized());
}
#[test]
fn test_runtime_shutdown() {
let rt = X86OpenMP::new();
rt.shutdown();
assert!(rt.is_shutting_down());
}
#[test]
fn test_runtime_gtid_allocation() {
let rt = X86OpenMP::new();
let a = rt.allocate_gtid();
let b = rt.allocate_gtid();
assert_ne!(a, b);
}
#[test]
fn test_runtime_region_stack() {
let rt = X86OpenMP::new();
let r1 = Arc::new(ParallelRegionState::new(0, 2));
let r2 = Arc::new(ParallelRegionState::new(0, 4));
rt.push_parallel_region(r1);
assert_eq!(rt.get_active_level(), 1);
rt.push_parallel_region(r2);
assert_eq!(rt.get_active_level(), 2);
rt.pop_parallel_region();
assert_eq!(rt.get_active_level(), 1);
rt.pop_parallel_region();
assert_eq!(rt.get_active_level(), 0);
}
#[test]
fn test_omp_get_thread_num_serial() {
init_test();
assert_eq!(omp_get_thread_num(), 0);
}
#[test]
fn test_omp_get_num_threads_serial() {
init_test();
assert_eq!(omp_get_num_threads(), 1);
}
#[test]
fn test_omp_in_parallel_serial() {
init_test();
assert_eq!(omp_in_parallel(), 0);
}
#[test]
fn test_omp_get_set_dynamic() {
init_test();
omp_set_dynamic(1);
assert_eq!(omp_get_dynamic(), 1);
}
#[test]
fn test_omp_get_set_nested() {
init_test();
omp_set_nested(1);
assert_eq!(omp_get_nested(), 1);
}
#[test]
fn test_omp_get_set_num_threads() {
init_test();
omp_set_num_threads(8);
assert_eq!(omp_get_max_threads(), 8);
}
#[test]
fn test_omp_get_num_procs() {
init_test();
assert!(omp_get_num_procs() > 0);
}
#[test]
fn test_omp_get_thread_limit() {
init_test();
assert_eq!(omp_get_thread_limit(), OMP_MAX_THREADS_X86 as i32);
}
#[test]
fn test_omp_get_max_active_levels() {
init_test();
assert_eq!(omp_get_max_active_levels(), OMP_MAX_ACTIVE_LEVELS as i32);
}
#[test]
fn test_omp_set_get_max_active_levels() {
init_test();
omp_set_max_active_levels(3);
assert_eq!(omp_get_max_active_levels(), 3);
}
#[test]
fn test_omp_get_level_serial() {
init_test();
assert_eq!(omp_get_level(), 0);
}
#[test]
fn test_omp_get_wtime() {
init_test();
let t1 = omp_get_wtime();
assert!(t1 >= 0.0);
let t2 = omp_get_wtime();
assert!(t2 >= t1);
}
#[test]
fn test_omp_get_wtick() {
assert!(omp_get_wtick() > 0.0);
}
#[test]
fn test_omp_get_set_schedule() {
init_test();
omp_set_schedule(schedule::SCHEDULE_GUIDED, 7);
let mut k = 0;
let mut m = 0;
omp_get_schedule(&mut k, &mut m);
assert_eq!(k, schedule::SCHEDULE_GUIDED);
assert_eq!(m, 7);
}
#[test]
fn test_omp_lock_full_api() {
let mut lock: *mut OMPLock = std::ptr::null_mut();
omp_init_lock(&mut lock);
assert!(omp_test_lock(&mut lock) == 1);
omp_unset_lock(&mut lock);
omp_set_lock(&mut lock);
assert!(omp_test_lock(&mut lock) == 0);
omp_unset_lock(&mut lock);
omp_destroy_lock(&mut lock);
assert!(lock.is_null());
}
#[test]
fn test_omp_nest_lock_full_api() {
let mut lock: *mut OMPNestLock = std::ptr::null_mut();
omp_init_nest_lock(&mut lock);
assert!(omp_test_nest_lock(&mut lock) > 0);
assert!(omp_test_nest_lock(&mut lock) > 0);
omp_unset_nest_lock(&mut lock);
omp_unset_nest_lock(&mut lock);
omp_destroy_nest_lock(&mut lock);
assert!(lock.is_null());
}
#[test]
fn test_omp_lock_null_safety() {
omp_destroy_lock(std::ptr::null_mut());
omp_set_lock(std::ptr::null_mut());
omp_unset_lock(std::ptr::null_mut());
assert_eq!(omp_test_lock(std::ptr::null_mut()), 0);
}
#[test]
fn test_target_register_unregister_noop() {
__tgt_register_lib(std::ptr::null());
__tgt_unregister_lib(std::ptr::null());
}
#[test]
fn test_target_returns_failure() {
assert_eq!(
__tgt_target(
0,
std::ptr::null(),
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut()
),
-1
);
assert_eq!(
__tgt_target_nowait(
0,
std::ptr::null(),
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut()
),
-1
);
}
#[test]
fn test_omp_target_alloc() {
let p = omp_target_alloc(64, 0);
assert!(!p.is_null());
omp_target_free(p, 0);
}
#[test]
fn test_omp_target_memcpy() {
let mut dst = [0u8; 8];
let src = [1, 2, 3, 4, 5, 6, 7, 8];
assert_eq!(
omp_target_memcpy(dst.as_mut_ptr(), src.as_ptr(), 8, 0, 0, 0, 0),
0
);
assert_eq!(dst, src);
}
#[test]
fn test_omp_is_initial_device() {
assert_eq!(omp_is_initial_device(), 1);
}
#[test]
fn test_omp_get_initial_device() {
assert_eq!(omp_get_initial_device(), 0);
}
#[test]
fn test_omp_get_num_devices() {
assert_eq!(omp_get_num_devices(), 0);
}
#[test]
fn test_omp_get_cancellation() {
assert_eq!(omp_get_cancellation(), 0);
}
#[test]
fn test_omp_get_proc_bind() {
assert_eq!(omp_get_proc_bind(), ProcBind::False as i32);
}
#[test]
fn test_omp_get_num_places() {
init_test();
assert!(omp_get_num_places() > 0);
}
#[test]
fn test_omp_get_max_task_priority() {
assert_eq!(omp_get_max_task_priority(), 1);
}
#[test]
fn test_omp_pause_resource_noop() {
assert_eq!(omp_pause_resource(0, 0), 0);
}
#[test]
fn test_omp_pause_resource_all_noop() {
assert_eq!(omp_pause_resource_all(0), 0);
}
#[test]
fn test_abi_kmpc_barrier() {
unsafe {
__kmpc_barrier(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_flush() {
unsafe {
__kmpc_flush(&Ident::default());
}
}
#[test]
fn test_abi_kmpc_global_thread_num() {
THREAD_GLOBAL_ID.with(|id| id.set(5));
assert_eq!(unsafe { __kmpc_global_thread_num(&Ident::default()) }, 5);
}
#[test]
fn test_abi_kmpc_simd() {
unsafe {
__kmpc_simd(&Ident::default());
}
}
#[test]
fn test_abi_kmpc_for_static_fini() {
unsafe {
__kmpc_for_static_fini(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_dispatch_fini_4() {
unsafe {
__kmpc_dispatch_fini_4(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_dispatch_fini_8() {
unsafe {
__kmpc_dispatch_fini_8(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_cancel_barrier() {
unsafe {
__kmpc_cancel_barrier(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_end_master() {
unsafe {
__kmpc_end_master(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_push_num_threads() {
init_test();
unsafe {
__kmpc_push_num_threads(&Ident::default(), 0, 12);
}
assert_eq!(omp_get_max_threads(), 12);
}
#[test]
fn test_abi_kmpc_ordered() {
init_test();
unsafe {
__kmpc_ordered(&Ident::default(), 0);
}
unsafe {
__kmpc_end_ordered(&Ident::default(), 0);
}
}
#[test]
fn test_abi_kmpc_taskloop() {
init_test();
unsafe extern "C" fn dummy(_g: i32, _t: *mut TaskDescriptor) {}
unsafe {
__kmpc_taskloop(&Ident::default(), 0, dummy, 0, 9, 1, 0, 5, 0);
}
}
#[test]
fn test_abi_kmpc_sections() {
init_test();
let rt = X86OpenMP::new();
with_region(&rt, 1, || {
let ident = Ident::default();
let ns = unsafe { __kmpc_sections_init(&ident, 0, 3) };
assert_eq!(ns, 3);
assert_eq!(unsafe { __kmpc_next_section(&ident, 0) }, 0);
assert_eq!(unsafe { __kmpc_next_section(&ident, 0) }, 1);
assert_eq!(unsafe { __kmpc_next_section(&ident, 0) }, 2);
assert_eq!(unsafe { __kmpc_next_section(&ident, 0) }, u32::MAX);
});
}
#[test]
fn test_for_static_init_4_with_region() {
let rt = X86OpenMP::new();
with_region(&rt, 4, || {
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 99i32;
let mut st = 1i32;
unsafe {
__kmpc_for_static_init_4(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
assert!(lo >= 0 && hi <= 99);
});
}
#[test]
fn test_for_static_init_8_with_region() {
let rt = X86OpenMP::new();
with_region(&rt, 4, || {
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i64;
let mut hi = 99i64;
let mut st = 1i64;
unsafe {
__kmpc_for_static_init_8(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
assert!(lo >= 0 && hi <= 99);
});
}
#[test]
fn test_for_static_init_4u_smoke() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0u32;
let mut hi = 99u32;
let mut st = 1i32;
unsafe {
__kmpc_for_static_init_4u(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
}
#[test]
fn test_for_static_init_8u_smoke() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0u64;
let mut hi = 99u64;
let mut st = 1i64;
unsafe {
__kmpc_for_static_init_8u(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
}
#[test]
fn test_dist_for_static_init_4_smoke() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 99i32;
let mut hi_dist = 0i32;
unsafe {
__kmpc_dist_for_static_init_4(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut hi_dist,
1,
0,
);
}
}
#[test]
fn test_dist_for_static_init_8_smoke() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i64;
let mut hi = 99i64;
let mut hi_dist = 0i64;
unsafe {
__kmpc_dist_for_static_init_8(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut hi_dist,
1,
0,
);
}
}
#[test]
fn test_dispatch_next_4_smoke() {
let rt = X86OpenMP::new();
with_region(&rt, 1, || {
let ident = Ident::default();
unsafe {
__kmpc_for_dynamic_init_4(&ident, 0, schedule::SCHEDULE_DYNAMIC, 0, 99, 1, 10);
}
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 0i32;
let mut st = 0i32;
let r =
unsafe { __kmpc_dispatch_next_4(&ident, 0, &mut last, &mut lo, &mut hi, &mut st) };
assert!(r == 0 || r == 1);
});
}
#[test]
fn test_dispatch_next_8_smoke() {
let rt = X86OpenMP::new();
with_region(&rt, 1, || {
let ident = Ident::default();
unsafe {
__kmpc_for_dynamic_init_8(&ident, 0, schedule::SCHEDULE_DYNAMIC, 0, 99, 1, 10);
}
let mut last = 0i32;
let mut lo = 0i64;
let mut hi = 0i64;
let mut st = 0i64;
let r =
unsafe { __kmpc_dispatch_next_8(&ident, 0, &mut last, &mut lo, &mut hi, &mut st) };
assert!(r == 0 || r == 1);
});
}
#[test]
fn test_omp_task_alloc_and_free() {
init_test();
unsafe extern "C" fn dummy(_g: i32, _t: *mut TaskDescriptor) {}
let ident = Ident::default();
let ptr = unsafe { __kmpc_omp_task_alloc(&ident, 0, 0, 64, 128, dummy) };
assert!(!ptr.is_null());
let task = unsafe { Box::from_raw(ptr) };
if !task.shareds.is_null() {
let layout = std::alloc::Layout::from_size_align(task.shareds_size, 8).unwrap();
unsafe { std::alloc::dealloc(task.shareds, layout) };
}
}
#[test]
fn test_omp_task_with_deps() {
init_test();
unsafe extern "C" fn dummy(_g: i32, _t: *mut TaskDescriptor) {}
let ident = Ident::default();
let ptr = unsafe { __kmpc_omp_task_alloc(&ident, 0, 0, 64, 0, dummy) };
let ret = unsafe {
__kmpc_omp_task_with_deps(&ident, 0, ptr, 2, std::ptr::null(), 0, std::ptr::null())
};
assert_eq!(ret, 0);
}
#[test]
fn test_omp_task_begin_complete_if0() {
init_test();
unsafe extern "C" fn dummy(_g: i32, _t: *mut TaskDescriptor) {}
let ident = Ident::default();
let ptr = unsafe { __kmpc_omp_task_alloc(&ident, 0, 0, 64, 0, dummy) };
unsafe {
__kmpc_omp_task_begin_if0(&ident, 0, ptr);
}
assert_eq!(unsafe { &*ptr }.status.load(Ordering::Acquire), 1);
unsafe {
__kmpc_omp_task_complete_if0(&ident, 0, ptr);
}
assert_eq!(unsafe { &*ptr }.status.load(Ordering::Acquire), 2);
}
#[test]
fn test_omp_taskwait_taskyield() {
init_test();
let ident = Ident::default();
unsafe {
__kmpc_omp_taskwait(&ident, 0);
}
unsafe {
__kmpc_omp_taskyield(&ident, 0);
}
}
#[test]
fn test_omp_taskgroup_begin_end() {
init_test();
let ident = Ident::default();
unsafe {
__kmpc_omp_taskgroup(&ident, 0);
}
unsafe {
__kmpc_omp_taskgroup_end(&ident, 0);
}
}
#[test]
fn test_omp_target_task_alloc() {
init_test();
unsafe extern "C" fn dummy(_g: i32, _t: *mut TaskDescriptor) {}
let ident = Ident::default();
let ptr = unsafe { __kmpc_omp_target_task_alloc(&ident, 0, 0, 64, 0, dummy) };
assert!(!ptr.is_null());
let task = unsafe { Box::from_raw(ptr) };
assert!(task.flags & TASK_FLAG_TARGET != 0);
}
#[test]
fn test_abi_kmpc_reduce() {
init_test();
let ident = Ident::default();
let mut data: i32 = 0;
unsafe extern "C" fn rf(_d: *mut u8, _s: *const u8) {}
let ret = unsafe {
__kmpc_reduce(
&ident,
0,
1,
4,
&mut data as *mut i32 as *mut u8,
rf,
std::ptr::null_mut(),
)
};
assert!(ret == 0 || ret == 1);
}
#[test]
fn test_abi_kmpc_end_reduce() {
unsafe {
__kmpc_end_reduce(&Ident::default(), 0, std::ptr::null_mut());
}
}
#[test]
fn test_kmpc_simd_width_functions() {
assert!(kmpc_simd_width_i32() >= 4);
assert!(kmpc_simd_width_i64() >= 2);
assert!(kmpc_simd_width_f32() >= 4);
assert!(kmpc_simd_width_f64() >= 2);
}
#[test]
fn test_guided_chunk_size() {
let sz = guided_chunk_size(100, 4);
assert!(sz >= OMP_GUIDED_MIN_CHUNK);
assert!(sz <= 25);
}
#[test]
fn test_default_ident() {
assert_eq!(default_ident().flags, 0);
}
#[test]
fn test_get_runtime() {
let _r = get_runtime();
}
#[test]
fn test_current_gtid() {
THREAD_GLOBAL_ID.with(|id| id.set(7));
assert_eq!(current_gtid(), 7);
}
#[test]
fn test_with_current_icv() {
init_test();
with_current_icv(|icv| {
icv.num_threads = 7;
});
assert_eq!(omp_get_max_threads(), 7);
}
#[test]
fn test_omp_display_env_no_crash() {
omp_display_env(0);
}
#[test]
fn test_omp_initialize_finalize() {
omp_initialize();
omp_finalize();
}
#[test]
fn test_constants_consistency() {
assert_eq!(OMP_MAX_THREADS_X86, 256);
assert!(TASK_FLAG_TARGET & 0x100 != 0);
assert_eq!(DEPEND_IN, 1);
assert_eq!(DEPEND_MUTEXINOUTSET, 4);
}
#[test]
fn test_compute_static_bounds_empty() {
let (lo, hi, last) = compute_static_bounds(0, 1, 4, 0, 0, 1, 0);
assert_eq!(lo, 0);
assert_eq!(hi, 0);
assert!(!last);
}
#[test]
fn test_taskloop_zero_iters() {
let s = TaskLoopState::new(1, 0, 1, 1, 0);
assert!(s.next_chunk().is_none());
}
#[test]
fn test_sections_zero_sections() {
let s = SectionsState::new(0);
assert!(s.next().is_none());
}
#[test]
fn test_barrier_one_thread() {
let b = TeamBarrier::new(1);
b.wait(); }
#[test]
fn test_fork_call_with_region_single() {
init_test();
omp_set_num_threads(1);
let ident = Ident::default();
unsafe extern "C" fn mt(_g: i32, _b: i32) {}
unsafe {
__kmpc_fork_call(&ident, 0, mt);
}
}
#[test]
fn test_static_bounds_chunked() {
let (lo, hi, last) = compute_static_bounds(100, 10, 4, 1, 0, 1, 99);
assert_eq!(lo, 10);
assert_eq!(hi, 19);
}
#[test]
fn test_static_bounds_offset() {
let (lo, hi, _) = compute_static_bounds(50, 1, 2, 1, 100, 1, 149);
assert_eq!(lo, 125);
assert_eq!(hi, 149);
}
#[test]
fn test_static_bounds_negative_stride() {
let (lo, hi, _) = compute_static_bounds(10, 1, 2, 0, 9, -1, 0);
assert_eq!(hi, 5);
}
#[test]
fn test_team_workshare_active_flag() {
let tw = TeamWorkshare::new();
assert!(!tw.active.load(Ordering::Acquire));
tw.active.store(true, Ordering::Release);
tw.reset();
assert!(!tw.active.load(Ordering::Acquire));
}
#[test]
fn test_parallel_region_reductions() {
let r = ParallelRegionState::new(0, 2);
let ctx1 = r.get_reduction(1);
let ctx2 = r.get_reduction(2);
let ctx1b = r.get_reduction(1);
assert!(Arc::ptr_eq(&ctx1, &ctx1b));
assert!(!Arc::ptr_eq(&ctx1, &ctx2));
}
#[test]
fn test_parallel_region_level_set() {
let r = ParallelRegionState::new(0, 2);
assert_eq!(r.level, 0);
assert!(!r.in_teams);
}
#[test]
fn test_x86ompp_is_master_outside_region() {
let rt = X86OpenMP::new();
assert!(rt.is_master()); }
#[test]
fn test_x86ompp_is_master_inside_region() {
let rt = X86OpenMP::new();
with_region(&rt, 2, || {
assert!(rt.is_master()); });
}
#[test]
fn test_debug_log_level() {
let rt = X86OpenMP::new();
rt.set_debug_level(1);
rt.debug_log(1, "should print");
rt.debug_log(2, "should not print");
rt.set_debug_level(0);
rt.debug_log(1, "quiet now");
}
#[test]
fn test_omp_target_memcpy_rect_basic() {
let mut dst = [0u8; 16];
let src = [1u8; 16];
let vol: usize = 16;
let r = omp_target_memcpy_rect(
dst.as_mut_ptr(),
src.as_ptr(),
4,
2,
&vol,
std::ptr::null(),
std::ptr::null(),
std::ptr::null(),
std::ptr::null(),
0,
0,
);
assert_eq!(r, 0);
assert_eq!(dst, src);
}
#[test]
fn test_omp_target_memcpy_rect_null() {
let r = omp_target_memcpy_rect(
std::ptr::null_mut(),
std::ptr::null(),
0,
0,
std::ptr::null(),
std::ptr::null(),
std::ptr::null(),
std::ptr::null(),
std::ptr::null(),
0,
0,
);
assert_eq!(r, -1);
}
#[test]
fn test_target_teams_stubs() {
assert_eq!(
__tgt_target_teams(
0,
std::ptr::null(),
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
1,
1
),
-1
);
assert_eq!(
__tgt_target_teams_nowait(
0,
std::ptr::null(),
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
1,
1
),
-1
);
}
#[test]
fn test_target_data_stubs() {
unsafe {
__tgt_target_data_begin(
0,
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
);
__tgt_target_data_end(
0,
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
);
__tgt_target_data_update(
0,
0,
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
std::ptr::null_mut(),
);
}
}
#[test]
fn test_omp_get_place_num_procs_range() {
init_test();
let n = omp_get_place_num_procs(0);
assert!(n >= 0);
assert_eq!(omp_get_place_num_procs(-1), 0);
assert_eq!(omp_get_place_num_procs(99999), 0);
}
#[test]
fn test_omp_get_place_proc_ids_null() {
omp_get_place_proc_ids(0, std::ptr::null_mut());
}
#[test]
fn test_omp_get_set_default_device() {
omp_set_default_device(0);
assert_eq!(omp_get_default_device(), 0);
}
#[test]
fn test_dispatch_init_guided_4() {
let rt = X86OpenMP::new();
with_region(&rt, 2, || {
let ident = Ident::default();
unsafe {
__kmpc_for_dynamic_init_4(&ident, 0, schedule::SCHEDULE_GUIDED, 0, 99, 1, 0);
}
});
}
#[test]
fn test_dispatch_init_guided_8() {
let rt = X86OpenMP::new();
with_region(&rt, 2, || {
let ident = Ident::default();
unsafe {
__kmpc_for_dynamic_init_8(&ident, 0, schedule::SCHEDULE_GUIDED, 0, 999, 1, 0);
}
});
}
#[test]
fn test_dispatch_next_guided_4() {
let rt = X86OpenMP::new();
with_region(&rt, 2, || {
let ident = Ident::default();
unsafe {
__kmpc_for_dynamic_init_4(&ident, 0, schedule::SCHEDULE_GUIDED, 0, 99, 1, 0);
}
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 0i32;
let mut st = 0i32;
let r =
unsafe { __kmpc_dispatch_next_4(&ident, 0, &mut last, &mut lo, &mut hi, &mut st) };
assert!(r == 0 || r == 1);
if r == 1 {
let first_size = hi - lo;
let r2 = unsafe {
__kmpc_dispatch_next_4(&ident, 0, &mut last, &mut lo, &mut hi, &mut st)
};
if r2 == 1 {
let second_size = hi - lo;
assert!(second_size <= first_size + 5); }
}
});
}
#[test]
fn test_runtime_get_gtid_default() {
init_test();
THREAD_GLOBAL_ID.with(|id| id.set(100));
let rt = X86OpenMP::new();
assert_eq!(rt.get_gtid(), 100);
}
#[test]
fn test_for_static_init_no_region() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 0i32;
let mut st = 1i32;
unsafe {
__kmpc_for_static_init_4(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
}
#[test]
fn test_fork_call_min_threads() {
init_test();
THREAD_ICV.with(|icv| icv.borrow_mut().num_threads = 0);
let ident = Ident::default();
unsafe extern "C" fn mt(_g: i32, _b: i32) {}
unsafe {
__kmpc_fork_call(&ident, 0, mt);
}
}
#[test]
fn test_lock_contention_count() {
let lock = OMPLock::new();
assert_eq!(lock.contention_count(), 0);
lock.set(0);
assert_eq!(lock.contention_count(), 0);
}
#[test]
fn test_nest_lock_depth() {
let lock = OMPNestLock::new();
assert_eq!(lock.depth(), 0);
lock.set(1);
assert_eq!(lock.depth(), 1);
lock.set(1);
assert_eq!(lock.depth(), 2);
lock.unset();
assert_eq!(lock.depth(), 1);
}
#[test]
fn test_task_flags() {
let mut t = TaskDescriptor::new(1, 0);
t.flags = TASK_FLAG_UNTIED;
assert!(t.is_untied());
assert!(!t.is_tied());
t.flags = TASK_FLAG_FINAL;
assert!(t.is_final());
t.flags = TASK_FLAG_TARGET;
assert!(t.is_target_task());
t.flags = TASK_FLAG_MERGABLE;
assert!(t.is_mergable());
}
#[test]
fn test_thread_pool_stack_size() {
let p = ThreadPool::new();
p.set_stack_size(1024 * 1024);
}
#[test]
fn test_with_current_icv_advanced() {
init_test();
let nested = with_current_icv(|icv| icv.nested);
assert!(!nested);
with_current_icv(|icv| icv.nested = true);
let nested = with_current_icv(|icv| icv.nested);
assert!(nested);
}
#[test]
fn test_omp_set_num_teams_noop() {
omp_set_num_teams(4);
omp_set_teams_thread_limit(128);
}
#[test]
fn test_single_end_reset_flag() {
init_test();
let rt = X86OpenMP::new();
let region = Arc::new(ParallelRegionState::new(0, 2));
rt.push_parallel_region(Arc::clone(®ion));
THREAD_GLOBAL_ID.with(|id| id.set(0));
region.single.try_acquire();
let ident = Ident::default();
unsafe {
__kmpc_end_single(&ident, 0);
}
assert!(!region.single.executed.load(Ordering::Acquire));
rt.pop_parallel_region();
}
#[test]
fn test_ordered_serialized() {
let rt = X86OpenMP::new();
let region = Arc::new(ParallelRegionState::new(0, 2));
rt.push_parallel_region(Arc::clone(®ion));
let ident = Ident::default();
unsafe {
__kmpc_ordered(&ident, 0);
}
let region2 = Arc::clone(®ion);
let h = std::thread::spawn(move || {
let ident = Ident::default();
unsafe {
__kmpc_ordered(&ident, 1);
}
unsafe {
__kmpc_end_ordered(&ident, 1);
}
});
std::thread::sleep(Duration::from_millis(10));
unsafe {
__kmpc_end_ordered(&ident, 0);
}
h.join().unwrap();
rt.pop_parallel_region();
}
#[test]
fn test_parallel_region_nesting_levels() {
let rt = X86OpenMP::new();
let r1 = Arc::new(ParallelRegionState::new(0, 2));
let r2 = Arc::new(ParallelRegionState::new(0, 4));
rt.push_parallel_region(r1);
assert_eq!(rt.current_parallel_region().unwrap().level, 0);
rt.push_parallel_region(r2);
assert_eq!(rt.current_parallel_region().unwrap().level, 1);
rt.pop_parallel_region();
rt.pop_parallel_region();
}
#[test]
fn test_critical_section_different_keys() {
let table = CriticalLockTable::new();
let a = table.get_or_create("key_a");
let b = table.get_or_create("key_b");
assert!(!Arc::ptr_eq(&a, &b));
}
#[test]
fn test_reduction_context_concurrent() {
let ctx = Arc::new(ReductionContext::new());
let ctx2 = Arc::clone(&ctx);
let h = std::thread::spawn(move || {
let _g = ctx2.lock.lock().unwrap();
ctx2.active.store(true, Ordering::Release);
});
h.join().unwrap();
assert!(ctx.active.load(Ordering::Acquire));
}
#[test]
fn test_dist_for_state_values() {
let mut df = DistForState::default();
df.chunk_lower[0] = 10;
df.chunk_upper[0] = 20;
df.num_chunks = 1;
df.initialized = true;
assert!(df.initialized);
assert_eq!(df.chunk_lower[0], 10);
}
#[test]
fn test_all_schedule_constants() {
assert_eq!(schedule::SCHEDULE_STATIC, 1);
assert_eq!(schedule::SCHEDULE_DYNAMIC, 2);
assert_eq!(schedule::SCHEDULE_GUIDED, 3);
assert_eq!(schedule::SCHEDULE_AUTO, 4);
assert_eq!(schedule::SCHEDULE_RUNTIME, 5);
assert!(schedule::SCHEDULE_MONOTONIC != 0);
assert!(schedule::SCHEDULE_NONMONOTONIC != 0);
assert_ne!(
schedule::SCHEDULE_MONOTONIC,
schedule::SCHEDULE_NONMONOTONIC
);
}
#[test]
fn test_barrier_patterns_distinct() {
assert_eq!(BARRIER_PATTERN_PARALLEL, 0);
assert_eq!(BARRIER_PATTERN_FOR, 1);
assert_eq!(BARRIER_PATTERN_SECTIONS, 2);
assert_eq!(BARRIER_PATTERN_SINGLE, 3);
assert_eq!(BARRIER_PATTERN_TASKGROUP, 4);
assert_eq!(BARRIER_PATTERN_REDUCTION, 5);
}
#[test]
fn test_cancellation_constants() {
assert_eq!(CANCEL_PARALLEL, 1);
assert_eq!(CANCEL_LOOP, 2);
assert_eq!(CANCEL_SECTIONS, 4);
assert_eq!(CANCEL_TASKGROUP, 8);
}
#[test]
fn test_dependency_type_constants() {
assert_eq!(DEPEND_IN, 1);
assert_eq!(DEPEND_OUT, 2);
assert_eq!(DEPEND_INOUT, 3);
assert_eq!(DEPEND_MUTEXINOUTSET, 4);
assert_eq!(DEPEND_SOURCE, 5);
assert_eq!(DEPEND_SINK, 6);
assert_eq!(DEPEND_DEPOBJ, 7);
}
#[test]
fn test_proc_bind_values() {
assert_eq!(ProcBind::False as i32, 0);
assert_eq!(ProcBind::True as i32, 1);
assert_eq!(ProcBind::Master as i32, 2);
assert_eq!(ProcBind::Close as i32, 3);
assert_eq!(ProcBind::Spread as i32, 4);
}
#[test]
fn test_wait_policy_equality() {
assert_eq!(WaitPolicy::Active, WaitPolicy::Active);
assert_ne!(WaitPolicy::Active, WaitPolicy::Passive);
}
#[test]
fn test_task_queue_stats() {
let q = TaskQueue::new();
assert_eq!(q.total_created(), 0);
assert_eq!(q.total_completed(), 0);
q.enqueue_task(Arc::new(TaskDescriptor::new(1, 0)));
assert_eq!(q.total_created(), 1);
assert_eq!(q.ready_count(), 1);
}
#[test]
fn test_reduction_table_len() {
let t = ReductionTable::new();
assert!(t.len() > 10);
assert!(!t.is_empty());
let keys = t.keys();
assert!(!keys.is_empty());
}
#[test]
fn test_region_cancellation() {
let r = ParallelRegionState::new(0, 2);
assert!(!r.cancelled.load(Ordering::Acquire));
r.cancelled.store(true, Ordering::Release);
}
#[test]
fn test_task_created_at() {
let before = Instant::now();
let t = TaskDescriptor::new(1, 0);
assert!(t.created_at >= before);
}
#[test]
fn test_lock_destroy_nullifies_pointer() {
let mut lock: *mut OMPLock = std::ptr::null_mut();
omp_init_lock(&mut lock);
omp_destroy_lock(&mut lock);
assert!(lock.is_null());
}
#[test]
fn test_nest_lock_destroy_nullifies_pointer() {
let mut lock: *mut OMPNestLock = std::ptr::null_mut();
omp_init_nest_lock(&mut lock);
omp_destroy_nest_lock(&mut lock);
assert!(lock.is_null());
}
#[test]
fn test_simd_i8_i16_counts() {
let sse = X86SimdWidth::SSE;
assert_eq!(sse.i8_count(), 16);
assert_eq!(sse.i16_count(), 8);
assert_eq!(sse.bit_width(), 128);
let avx = X86SimdWidth::AVX;
assert_eq!(avx.i8_count(), 32);
assert_eq!(avx.i16_count(), 16);
let avx512 = X86SimdWidth::AVX512;
assert_eq!(avx512.i8_count(), 64);
assert_eq!(avx512.i16_count(), 32);
}
#[test]
fn test_simd_width_ordering() {
assert!(X86SimdWidth::SSE < X86SimdWidth::AVX);
assert!(X86SimdWidth::AVX < X86SimdWidth::AVX512);
}
#[test]
fn test_kmpc_simd_width_i8() {
let w = kmpc_simd_width_i8();
assert!(w >= 16);
assert!(w == 16 || w == 32 || w == 64);
}
#[test]
fn test_kmpc_simd_width_i16() {
let w = kmpc_simd_width_i16();
assert!(w >= 8);
assert!(w == 8 || w == 16 || w == 32);
}
#[test]
fn test_icv_stack_save_restore() {
let mut ts = TaskExecutionState::new();
ts.icv_stack.push(ICVStorage::default());
ts.icv_stack.push(ICVStorage::default());
assert_eq!(ts.icv_stack.len(), 2);
ts.icv_stack.pop();
assert_eq!(ts.icv_stack.len(), 1);
}
#[test]
fn test_thread_task_state_defaults() {
let ts = TaskExecutionState::default();
assert_eq!(ts.current_task_id, 0);
assert!(!ts.in_taskgroup);
assert_eq!(ts.taskgroup_depth, 0);
assert_eq!(ts.tasks_created, 0);
}
#[test]
fn test_reduction_ops_all_present() {
let t = ReductionTable::new();
let required = &[
"i32_add", "i32_mul", "i32_and", "i32_or", "i32_xor", "i32_band", "i32_bor", "i32_min",
"i32_max", "i64_add", "i64_mul", "i64_and", "i64_or", "i64_min", "i64_max", "f32_add",
"f32_mul", "f32_min", "f32_max", "f64_add", "f64_mul", "f64_min", "f64_max", "u32_add",
"u32_min", "u32_max", "u64_add", "u64_min", "u64_max", "i8_add", "i8_min", "i8_max",
"i16_add",
];
for key in required {
assert!(t.get(key).is_some(), "Missing reduction: {}", key);
}
}
#[test]
fn test_reduction_op_values() {
assert_eq!(ReductionOp::Add as i32, 0);
assert_eq!(ReductionOp::Mul as i32, 1);
assert_eq!(ReductionOp::And as i32, 2);
assert_eq!(ReductionOp::Or as i32, 3);
assert_eq!(ReductionOp::Xor as i32, 4);
assert_eq!(ReductionOp::BAnd as i32, 5);
assert_eq!(ReductionOp::BOr as i32, 6);
assert_eq!(ReductionOp::Max as i32, 7);
assert_eq!(ReductionOp::Min as i32, 8);
assert_eq!(ReductionOp::Custom as i32, 9);
}
#[test]
fn test_barrier_set_pattern() {
let b = TeamBarrier::new(2);
b.set_pattern(BARRIER_PATTERN_FOR);
assert_eq!(b.pattern.load(Ordering::Acquire), BARRIER_PATTERN_FOR);
b.reset(4);
assert_eq!(b.pattern.load(Ordering::Acquire), BARRIER_PATTERN_PARALLEL);
}
#[test]
fn test_barrier_usage_count() {
let b = TeamBarrier::new(2);
assert_eq!(b.usage_count.load(Ordering::Relaxed), 0);
b.wait();
b.wait();
assert_eq!(b.usage_count.load(Ordering::Relaxed), 1);
}
#[test]
fn test_barrier_arrived_count() {
let b = TeamBarrier::new(3);
assert_eq!(b.arrived_count(), 0);
let b2 = Arc::new(b);
}
#[test]
fn test_runtime_env_parsed_flag() {
let rt = X86OpenMP::new();
assert!(!rt.env_parsed.load(Ordering::Acquire));
rt.initialize();
assert!(rt.env_parsed.load(Ordering::Acquire));
}
#[test]
fn test_runtime_debug_level() {
let rt = X86OpenMP::new();
assert_eq!(rt.debug_level.load(Ordering::Relaxed), 0);
rt.set_debug_level(3);
assert_eq!(rt.debug_level.load(Ordering::Relaxed), 3);
}
#[test]
fn test_icv_storage_clone() {
let icv1 = ICVStorage {
num_threads: 8,
thread_limit: 128,
dynamic: true,
nested: true,
schedule_kind: 3,
schedule_chunk: 5,
max_active_levels: 4,
in_parallel: true,
active_level: 2,
place_partition: vec![0, 1],
proc_bind: ProcBind::Close,
cancellation: true,
stack_size: 1_000_000,
wait_policy: WaitPolicy::Passive,
default_schedule: 2,
default_chunk: 3,
default_device: 1,
};
let icv2 = icv1.clone();
assert_eq!(icv2.num_threads, 8);
assert_eq!(icv2.thread_limit, 128);
assert!(icv2.dynamic);
assert!(icv2.nested);
assert_eq!(icv2.schedule_kind, 3);
assert_eq!(icv2.schedule_chunk, 5);
assert_eq!(icv2.max_active_levels, 4);
assert_eq!(icv2.place_partition, vec![0, 1]);
assert_eq!(icv2.proc_bind, ProcBind::Close);
assert!(icv2.cancellation);
assert_eq!(icv2.stack_size, 1_000_000);
assert_eq!(icv2.wait_policy, WaitPolicy::Passive);
assert_eq!(icv2.default_device, 1);
}
#[test]
fn test_static_bounds_remainder_accurate() {
let (lo0, hi0, _) = compute_static_bounds(7, 1, 3, 0, 0, 1, 6);
let (lo1, hi1, _) = compute_static_bounds(7, 1, 3, 1, 0, 1, 6);
let (lo2, hi2, _) = compute_static_bounds(7, 1, 3, 2, 0, 1, 6);
assert_eq!(lo0, 0);
assert_eq!(hi0, 2);
assert_eq!(lo1, 3);
assert_eq!(hi1, 4);
assert_eq!(lo2, 5);
assert_eq!(hi2, 6);
}
#[test]
fn test_task_status_transitions() {
let t = TaskDescriptor::new(1, 0);
assert_eq!(t.status.load(Ordering::Acquire), 0);
t.status.store(1, Ordering::Release);
assert_eq!(t.status.load(Ordering::Acquire), 1);
t.status.store(2, Ordering::Release);
assert_eq!(t.status.load(Ordering::Acquire), 2);
}
}
pub mod ompt_callbacks {
pub type OmptThreadBegin = Option<unsafe extern "C" fn(thr_type: i32, thr_id: u64)>;
pub type OmptThreadEnd = Option<unsafe extern "C" fn(thr_id: u64)>;
pub type OmptParallelBegin = Option<
unsafe extern "C" fn(
parent_task_data: *const u8,
encountering_frame: *const u8,
parallel_data: *const u8,
requested_parallelism: u32,
flags: u32,
),
>;
pub type OmptParallelEnd =
Option<unsafe extern "C" fn(parallel_data: *const u8, task_data: *const u8, flags: u32)>;
pub type OmptTaskCreate = Option<
unsafe extern "C" fn(
parent_task_data: *const u8,
encountering_frame: *const u8,
new_task_data: *const u8,
task_type: i32,
has_dependences: i32,
),
>;
pub type OmptTaskSchedule = Option<
unsafe extern "C" fn(
prior_task_data: *const u8,
prior_task_status: i32,
next_task_data: *const u8,
),
>;
pub type OmptImplicitTask =
Option<unsafe extern "C" fn(parallel_data: *const u8, task_data: *const u8, flags: u32)>;
pub type OmptWork = Option<
unsafe extern "C" fn(
wstype: i32,
endpoint: i32,
parallel_data: *const u8,
task_data: *const u8,
count: u64,
),
>;
pub type OmptSyncRegion = Option<
unsafe extern "C" fn(
kind: i32,
endpoint: i32,
parallel_data: *const u8,
task_data: *const u8,
),
>;
pub type OmptMutex = Option<unsafe extern "C" fn(kind: i32, wait_id: u64, codeptr: *const u8)>;
pub type OmptReduction = Option<
unsafe extern "C" fn(
parallel_data: *const u8,
task_data: *const u8,
reduction_op: i32,
buffer: *const u8,
),
>;
pub type OmptTarget = Option<
unsafe extern "C" fn(
kind: i32,
endpoint: i32,
device_num: i32,
task_data: *const u8,
target_id: u64,
host_op_id: u64,
buf: *const u8,
bytes: usize,
),
>;
}
pub struct OMPTCallbackTable {
pub thread_begin: ompt_callbacks::OmptThreadBegin,
pub thread_end: ompt_callbacks::OmptThreadEnd,
pub parallel_begin: ompt_callbacks::OmptParallelBegin,
pub parallel_end: ompt_callbacks::OmptParallelEnd,
pub task_create: ompt_callbacks::OmptTaskCreate,
pub task_schedule: ompt_callbacks::OmptTaskSchedule,
pub implicit_task_begin: ompt_callbacks::OmptImplicitTask,
pub implicit_task_end: ompt_callbacks::OmptImplicitTask,
pub work_begin: ompt_callbacks::OmptWork,
pub work_end: ompt_callbacks::OmptWork,
pub sync_region_begin: ompt_callbacks::OmptSyncRegion,
pub sync_region_end: ompt_callbacks::OmptSyncRegion,
pub mutex_acquired: ompt_callbacks::OmptMutex,
pub mutex_released: ompt_callbacks::OmptMutex,
pub reduction: ompt_callbacks::OmptReduction,
pub target: ompt_callbacks::OmptTarget,
}
impl OMPTCallbackTable {
pub fn new() -> Self {
OMPTCallbackTable {
thread_begin: None,
thread_end: None,
parallel_begin: None,
parallel_end: None,
task_create: None,
task_schedule: None,
implicit_task_begin: None,
implicit_task_end: None,
work_begin: None,
work_end: None,
sync_region_begin: None,
sync_region_end: None,
mutex_acquired: None,
mutex_released: None,
reduction: None,
target: None,
}
}
pub fn has_any(&self) -> bool {
self.thread_begin.is_some()
|| self.thread_end.is_some()
|| self.parallel_begin.is_some()
|| self.parallel_end.is_some()
|| self.task_create.is_some()
|| self.task_schedule.is_some()
|| self.implicit_task_begin.is_some()
|| self.implicit_task_end.is_some()
|| self.work_begin.is_some()
|| self.work_end.is_some()
|| self.sync_region_begin.is_some()
|| self.sync_region_end.is_some()
|| self.mutex_acquired.is_some()
|| self.mutex_released.is_some()
|| self.reduction.is_some()
|| self.target.is_some()
}
}
impl Default for OMPTCallbackTable {
fn default() -> Self {
OMPTCallbackTable::new()
}
}
pub static OMPT_CALLBACKS: std::sync::LazyLock<Mutex<OMPTCallbackTable>> =
std::sync::LazyLock::new(|| Mutex::new(OMPTCallbackTable::new()));
#[unsafe(no_mangle)]
pub extern "C" fn ompt_start_tool(_version: u32, _ompt_version: *const u8) -> i32 {
0
}
#[derive(Debug, Clone)]
pub enum OMPEvent {
ParallelBegin {
gtid: i32,
num_threads: u32,
level: u32,
},
ParallelEnd { gtid: i32, level: u32 },
TaskCreate {
task_id: u64,
parent_gtid: i32,
flags: i32,
},
TaskBegin { task_id: u64, gtid: i32 },
TaskComplete { task_id: u64, gtid: i32 },
WorkBegin { wstype: i32, lower: i64, upper: i64 },
WorkEnd { wstype: i32 },
BarrierBegin { gtid: i32, pattern: u32 },
BarrierEnd { gtid: i32 },
CriticalBegin { gtid: i32 },
CriticalEnd { gtid: i32 },
LockAcquire { gtid: i32 },
LockRelease { gtid: i32 },
ReductionBegin { gtid: i32 },
ReductionEnd { gtid: i32 },
ThreadCreate { gtid: i32 },
ThreadExit { gtid: i32 },
}
pub struct EventTrace {
events: Mutex<VecDeque<(OMPEvent, Instant)>>,
capacity: usize,
enabled: AtomicBool,
}
impl EventTrace {
pub fn new(capacity: usize) -> Self {
EventTrace {
events: Mutex::new(VecDeque::with_capacity(capacity)),
capacity,
enabled: AtomicBool::new(false),
}
}
pub fn enable(&self) {
self.enabled.store(true, Ordering::Release);
}
pub fn disable(&self) {
self.enabled.store(false, Ordering::Release);
}
pub fn is_enabled(&self) -> bool {
self.enabled.load(Ordering::Acquire)
}
pub fn record(&self, event: OMPEvent) {
if !self.is_enabled() {
return;
}
let mut events = self.events.lock().unwrap();
if events.len() >= self.capacity {
events.pop_front();
}
events.push_back((event, Instant::now()));
}
pub fn snapshot(&self) -> Vec<(OMPEvent, Instant)> {
let events = self.events.lock().unwrap();
events.iter().cloned().collect()
}
pub fn clear(&self) {
self.events.lock().unwrap().clear();
}
pub fn len(&self) -> usize {
self.events.lock().unwrap().len()
}
pub fn is_empty(&self) -> bool {
self.events.lock().unwrap().is_empty()
}
}
impl Default for EventTrace {
fn default() -> Self {
EventTrace::new(1024)
}
}
pub static EVENT_TRACE: std::sync::LazyLock<EventTrace> =
std::sync::LazyLock::new(|| EventTrace::new(4096));
#[derive(Debug, Clone)]
pub struct CpuTopology {
pub num_hw_threads: u32,
pub num_cores: u32,
pub num_sockets: u32,
pub threads_per_core: u32,
pub smt_active: bool,
}
impl CpuTopology {
pub fn detect() -> Self {
let num_hw_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1) as u32;
let num_cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1) as u32;
let threads_per_core = if num_cores > 0 {
num_hw_threads / num_cores
} else {
1
};
CpuTopology {
num_hw_threads,
num_cores,
num_sockets: 1, threads_per_core,
smt_active: threads_per_core > 1,
}
}
pub fn default_places(&self) -> Vec<Vec<i32>> {
match () {
_ => {
(0..self.num_hw_threads as i32).map(|i| vec![i]).collect()
}
}
}
pub fn core_places(&self) -> Vec<Vec<i32>> {
let mut places = Vec::new();
for core in 0..self.num_cores {
let mut core_threads = Vec::new();
for t in 0..self.threads_per_core {
core_threads.push((core * self.threads_per_core + t) as i32);
}
places.push(core_threads);
}
places
}
pub fn socket_places(&self) -> Vec<Vec<i32>> {
let mut places = Vec::new();
let cores_per_socket = if self.num_sockets > 0 {
self.num_cores / self.num_sockets
} else {
self.num_cores
};
for socket in 0..self.num_sockets {
let mut socket_threads = Vec::new();
for core in 0..cores_per_socket {
let base = socket * cores_per_socket * self.threads_per_core;
for t in 0..self.threads_per_core {
socket_threads.push((base + core * self.threads_per_core + t) as i32);
}
}
places.push(socket_threads);
}
places
}
}
pub struct ThreadPlacer {
pub topology: CpuTopology,
pub places: Vec<Vec<i32>>,
next_place: AtomicU32,
}
impl ThreadPlacer {
pub fn new() -> Self {
let topo = CpuTopology::detect();
let places = topo.default_places();
ThreadPlacer {
topology: topo,
places,
next_place: AtomicU32::new(0),
}
}
pub fn assign_place(&self, policy: ProcBind, gtid: i32) -> Option<Vec<i32>> {
if self.places.is_empty() {
return None;
}
match policy {
ProcBind::True | ProcBind::Close => {
let idx = self.next_place.fetch_add(1, Ordering::SeqCst);
let idx = (idx as usize) % self.places.len();
Some(self.places[idx].clone())
}
ProcBind::Spread => {
let stride = self.places.len().max(1) as u32;
let idx = (gtid as u32 * stride) % self.places.len() as u32;
Some(self.places[idx as usize].clone())
}
ProcBind::Master => {
Some(self.places[0].clone())
}
ProcBind::False => {
None }
}
}
pub fn num_places(&self) -> usize {
self.places.len()
}
pub fn place_procs(&self, place_num: usize) -> Option<&[i32]> {
self.places.get(place_num).map(|v| v.as_slice())
}
}
impl Default for ThreadPlacer {
fn default() -> Self {
ThreadPlacer::new()
}
}
#[derive(Debug)]
pub struct RuntimeStats {
pub parallel_regions: AtomicU64,
pub tasks_created: AtomicU64,
pub tasks_completed: AtomicU64,
pub barriers: AtomicU64,
pub lock_acquires: AtomicU64,
pub lock_contentions: AtomicU64,
pub workshare_loops: AtomicU64,
pub critical_sections: AtomicU64,
pub single_regions: AtomicU64,
pub reductions: AtomicU64,
pub dispatch_calls: AtomicU64,
pub parallel_time_ns: AtomicU64,
pub barrier_time_ns: AtomicU64,
pub peak_threads: AtomicU32,
}
impl RuntimeStats {
pub fn new() -> Self {
RuntimeStats {
parallel_regions: AtomicU64::new(0),
tasks_created: AtomicU64::new(0),
tasks_completed: AtomicU64::new(0),
barriers: AtomicU64::new(0),
lock_acquires: AtomicU64::new(0),
lock_contentions: AtomicU64::new(0),
workshare_loops: AtomicU64::new(0),
critical_sections: AtomicU64::new(0),
single_regions: AtomicU64::new(0),
reductions: AtomicU64::new(0),
dispatch_calls: AtomicU64::new(0),
parallel_time_ns: AtomicU64::new(0),
barrier_time_ns: AtomicU64::new(0),
peak_threads: AtomicU32::new(0),
}
}
pub fn reset(&self) {
self.parallel_regions.store(0, Ordering::Release);
self.tasks_created.store(0, Ordering::Release);
self.tasks_completed.store(0, Ordering::Release);
self.barriers.store(0, Ordering::Release);
self.lock_acquires.store(0, Ordering::Release);
self.lock_contentions.store(0, Ordering::Release);
self.workshare_loops.store(0, Ordering::Release);
self.critical_sections.store(0, Ordering::Release);
self.single_regions.store(0, Ordering::Release);
self.reductions.store(0, Ordering::Release);
self.dispatch_calls.store(0, Ordering::Release);
self.parallel_time_ns.store(0, Ordering::Release);
self.barrier_time_ns.store(0, Ordering::Release);
self.peak_threads.store(0, Ordering::Release);
}
pub fn report(&self) -> String {
format!(
"OMP Runtime Statistics:\n\
Parallel regions: {}\n\
Tasks created: {}\n\
Tasks completed: {}\n\
Barriers: {}\n\
Lock acquires: {}\n\
Lock contentions: {}\n\
Workshare loops: {}\n\
Critical sections: {}\n\
Single regions: {}\n\
Reductions: {}\n\
Dispatch calls: {}\n\
Parallel time: {:.3} ms\n\
Barrier time: {:.3} ms\n\
Peak threads: {}",
self.parallel_regions.load(Ordering::Relaxed),
self.tasks_created.load(Ordering::Relaxed),
self.tasks_completed.load(Ordering::Relaxed),
self.barriers.load(Ordering::Relaxed),
self.lock_acquires.load(Ordering::Relaxed),
self.lock_contentions.load(Ordering::Relaxed),
self.workshare_loops.load(Ordering::Relaxed),
self.critical_sections.load(Ordering::Relaxed),
self.single_regions.load(Ordering::Relaxed),
self.reductions.load(Ordering::Relaxed),
self.dispatch_calls.load(Ordering::Relaxed),
self.parallel_time_ns.load(Ordering::Relaxed) as f64 / 1_000_000.0,
self.barrier_time_ns.load(Ordering::Relaxed) as f64 / 1_000_000.0,
self.peak_threads.load(Ordering::Relaxed),
)
}
}
impl Default for RuntimeStats {
fn default() -> Self {
RuntimeStats::new()
}
}
pub static RUNTIME_STATS: std::sync::LazyLock<RuntimeStats> =
std::sync::LazyLock::new(|| RuntimeStats::new());
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(i32)]
pub enum OMPError {
Success = 0,
OutOfMemory = 1,
InvalidArgument = 2,
ThreadLimitExceeded = 3,
NestingLimitExceeded = 4,
LockNotInitialized = 5,
LockOwned = 6,
DependencyCycle = 7,
TargetOffloadFailed = 8,
Unsupported = 9,
Internal = 99,
}
impl OMPError {
pub fn as_str(&self) -> &'static str {
match self {
OMPError::Success => "success",
OMPError::OutOfMemory => "out of memory",
OMPError::InvalidArgument => "invalid argument",
OMPError::ThreadLimitExceeded => "thread limit exceeded",
OMPError::NestingLimitExceeded => "nesting limit exceeded",
OMPError::LockNotInitialized => "lock not initialized",
OMPError::LockOwned => "lock already owned",
OMPError::DependencyCycle => "dependency cycle",
OMPError::TargetOffloadFailed => "target offload failed",
OMPError::Unsupported => "unsupported operation",
OMPError::Internal => "internal error",
}
}
}
pub type OMPErrorHandler = Option<fn(error: OMPError, message: &str) -> !>;
static FATAL_ERROR_HANDLER: std::sync::LazyLock<Mutex<OMPErrorHandler>> =
std::sync::LazyLock::new(|| Mutex::new(None));
pub fn set_fatal_error_handler(handler: fn(OMPError, &str) -> !) {
*FATAL_ERROR_HANDLER.lock().unwrap() = Some(handler);
}
pub fn fatal_error(error: OMPError, msg: &str) -> ! {
let handler = *FATAL_ERROR_HANDLER.lock().unwrap();
match handler {
Some(h) => h(error, msg),
None => {
eprintln!("FATAL OpenMP runtime error [{}]: {}", error.as_str(), msg);
std::process::abort();
}
}
}
pub fn warning(msg: &str) {
eprintln!("OMP warning: {}", msg);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86MicroArch {
Unknown,
Nehalem,
SandyBridge,
IvyBridge,
Haswell,
Broadwell,
Skylake,
KabyLake,
CoffeeLake,
CannonLake,
IceLake,
TigerLake,
AlderLake,
RaptorLake,
SapphireRapids,
GraniteRapids,
Zen1,
Zen2,
Zen3,
Zen4,
Zen5,
}
impl X86MicroArch {
pub fn detect() -> Self {
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
{
if is_x86_feature_detected!("avx512f") {
if is_x86_feature_detected!("avx512bf16") {
return X86MicroArch::SapphireRapids;
}
return X86MicroArch::IceLake;
}
if is_x86_feature_detected!("avx2") {
if is_x86_feature_detected!("vaes") {
return X86MicroArch::IceLake;
}
return X86MicroArch::Haswell;
}
if is_x86_feature_detected!("avx") {
return X86MicroArch::SandyBridge;
}
X86MicroArch::Nehalem
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
{
X86MicroArch::Unknown
}
}
pub fn preferred_wait_policy(&self) -> WaitPolicy {
match self {
X86MicroArch::AlderLake | X86MicroArch::RaptorLake => WaitPolicy::Passive,
_ => WaitPolicy::Active,
}
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_version() -> i32 {
201511
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_api_version() -> i32 {
202111 }
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_supported_active_levels() -> i32 {
OMP_MAX_ACTIVE_LEVELS as i32
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_get_device_num() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_is_in_teams_region() -> i32 {
let runtime = &*GLOBAL_RUNTIME;
match runtime.current_parallel_region() {
Some(region) => {
if region.in_teams {
1
} else {
0
}
}
None => 0,
}
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_is_in_target_region() -> i32 {
0
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_alloc(size: usize, _allocator: *const u8) -> *mut u8 {
if size == 0 {
return std::ptr::null_mut();
}
let layout = std::alloc::Layout::from_size_align(size, 8).unwrap();
unsafe { std::alloc::alloc(layout) }
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_free(ptr: *mut u8, _allocator: *const u8) {
let _ = ptr;
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_calloc(nmemb: usize, size: usize, _allocator: *const u8) -> *mut u8 {
let total = nmemb.checked_mul(size).unwrap_or(0);
if total == 0 {
return std::ptr::null_mut();
}
let layout = std::alloc::Layout::from_size_align(total, 8).unwrap();
let ptr = unsafe { std::alloc::alloc_zeroed(layout) };
ptr
}
#[unsafe(no_mangle)]
pub extern "C" fn omp_realloc(
ptr: *mut u8,
size: usize,
_allocator: *const u8,
_free_allocator: *const u8,
) -> *mut u8 {
if size == 0 {
return std::ptr::null_mut();
}
let new_layout = std::alloc::Layout::from_size_align(size, 8).unwrap();
unsafe { std::alloc::realloc(ptr, new_layout, size) }
}
#[unsafe(no_mangle)]
pub static omp_default_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_large_cap_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_const_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_high_bw_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_low_lat_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_cgroup_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_pteam_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub static omp_thread_mem_alloc: u8 = 0;
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_alloc_shared(size: usize) -> *mut u8 { unsafe {
if size == 0 {
return std::ptr::null_mut();
}
let layout = std::alloc::Layout::from_size_align(size, 64).unwrap();
std::alloc::alloc(layout)
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_free_shared(ptr: *mut u8, _size: usize) {
let _ = ptr;
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_taskloop_task(
loc: *const Ident,
gtid: i32,
task_func: unsafe extern "C" fn(i32, *mut TaskDescriptor),
lower: i64,
upper: i64,
stride: i64,
grainsize: i64,
num_tasks: i64,
flags: i32,
task_alloc_fn: unsafe extern "C" fn(i32, *mut TaskDescriptor) -> *mut TaskDescriptor,
task_free_fn: unsafe extern "C" fn(*mut TaskDescriptor),
) { unsafe {
__kmpc_taskloop(
loc, gtid, task_func, lower, upper, stride, grainsize, num_tasks, flags,
);
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_omp_task_modify_flags(
task_ptr: *mut TaskDescriptor,
flags_to_set: i32,
flags_to_clear: i32,
) -> i32 { unsafe {
if task_ptr.is_null() {
return -1;
}
let task = &mut *task_ptr;
task.flags |= flags_to_set;
task.flags &= !flags_to_clear;
task.flags
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_init_4(
loc: *const Ident,
gtid: i32,
sched: i32,
lower: i32,
upper: i32,
stride: i32,
chunk: i32,
) { unsafe {
__kmpc_for_dynamic_init_4(loc, gtid, sched, lower, upper, stride, chunk);
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_dispatch_init_8(
loc: *const Ident,
gtid: i32,
sched: i32,
lower: i64,
upper: i64,
stride: i64,
chunk: i64,
) { unsafe {
__kmpc_for_dynamic_init_8(loc, gtid, sched, lower, upper, stride, chunk);
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_next(
_loc: *const Ident,
_gtid: i32,
plastiter: *mut i32,
plower: *mut i32,
pupper: *mut i32,
_pstride: *mut i32,
) -> i32 {
if !plastiter.is_null() {
unsafe {
*plastiter = 0;
}
}
0
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_fork_teams(
_loc: *const Ident,
_argc: i32,
microtask: unsafe extern "C" fn(i32, i32),
) { unsafe {
let runtime = &*GLOBAL_RUNTIME;
let gtid = runtime.get_gtid();
let num_teams = 1u32;
for team in 0..num_teams {
let num_threads = runtime.get_num_threads();
let actual_threads = num_threads.min(OMP_MAX_THREADS_X86).max(1);
let region = Arc::new(ParallelRegionState::new(gtid, actual_threads));
unsafe {
let region_ptr = Arc::as_ptr(®ion) as *mut ParallelRegionState;
(*region_ptr).in_teams = true;
}
runtime.push_parallel_region(Arc::clone(®ion));
microtask(gtid, 0);
runtime.pop_parallel_region();
}
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_4_simple(
loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut i32,
pupper: *mut i32,
pstride: *mut i32,
incr: i32,
chunk: i32,
) { unsafe {
__kmpc_for_static_init_4(
loc, gtid, sched, plastiter, plower, pupper, pstride, incr, chunk,
);
}}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __kmpc_for_static_init_8_simple(
loc: *const Ident,
gtid: i32,
sched: i32,
plastiter: *mut i32,
plower: *mut i64,
pupper: *mut i64,
pstride: *mut i64,
incr: i64,
chunk: i64,
) { unsafe {
__kmpc_for_static_init_8(
loc, gtid, sched, plastiter, plower, pupper, pstride, incr, chunk,
);
}}
#[unsafe(no_mangle)]
pub extern "C" fn omp_control_tool(_command: i32, _modifier: i32, _arg: *mut u8) -> i32 {
-1
}
#[cfg(test)]
mod extended_tests {
use super::*;
#[test]
fn test_ompt_callback_table_empty() {
let t = OMPTCallbackTable::new();
assert!(!t.has_any());
}
#[test]
fn test_ompt_callback_table_default() {
let t = OMPTCallbackTable::default();
assert!(!t.has_any());
}
#[test]
fn test_ompt_start_tool_returns_zero() {
assert_eq!(ompt_start_tool(0, std::ptr::null()), 0);
}
#[test]
fn test_event_trace_basic() {
let t = EventTrace::new(10);
assert!(t.is_empty());
t.enable();
assert!(t.is_enabled());
t.record(OMPEvent::ParallelBegin {
gtid: 0,
num_threads: 4,
level: 1,
});
assert_eq!(t.len(), 1);
t.disable();
t.record(OMPEvent::ParallelEnd { gtid: 0, level: 1 });
assert_eq!(t.len(), 1); }
#[test]
fn test_event_trace_capacity() {
let t = EventTrace::new(3);
t.enable();
for i in 0..5 {
t.record(OMPEvent::BarrierBegin {
gtid: i,
pattern: 0,
});
}
assert_eq!(t.len(), 3);
}
#[test]
fn test_event_trace_snapshot() {
let t = EventTrace::new(10);
t.enable();
t.record(OMPEvent::TaskCreate {
task_id: 1,
parent_gtid: 0,
flags: 0,
});
let snap = t.snapshot();
assert_eq!(snap.len(), 1);
}
#[test]
fn test_event_trace_clear() {
let t = EventTrace::new(10);
t.enable();
t.record(OMPEvent::LockAcquire { gtid: 0 });
t.clear();
assert!(t.is_empty());
}
#[test]
fn test_event_trace_default_enabled() {
assert!(!EVENT_TRACE.is_enabled());
}
#[test]
fn test_cpu_topology_detect() {
let topo = CpuTopology::detect();
assert!(topo.num_hw_threads > 0);
assert!(topo.num_cores > 0);
assert!(topo.threads_per_core >= 1);
}
#[test]
fn test_cpu_topology_default_places() {
let topo = CpuTopology::detect();
let places = topo.default_places();
assert_eq!(places.len() as u32, topo.num_hw_threads);
}
#[test]
fn test_cpu_topology_core_places() {
let topo = CpuTopology::detect();
let places = topo.core_places();
assert_eq!(places.len() as u32, topo.num_cores);
}
#[test]
fn test_cpu_topology_socket_places() {
let topo = CpuTopology::detect();
let places = topo.socket_places();
assert!(!places.is_empty());
}
#[test]
fn test_thread_placer_default() {
let p = ThreadPlacer::default();
assert!(p.num_places() > 0);
}
#[test]
fn test_thread_placer_assign_spread() {
let p = ThreadPlacer::new();
if p.num_places() > 0 {
let place = p.assign_place(ProcBind::Spread, 5);
assert!(place.is_some());
}
}
#[test]
fn test_thread_placer_assign_close() {
let p = ThreadPlacer::new();
if p.num_places() > 0 {
let p1 = p.assign_place(ProcBind::Close, 0);
let p2 = p.assign_place(ProcBind::Close, 1);
assert!(p1.is_some());
assert!(p2.is_some());
}
}
#[test]
fn test_thread_placer_assign_master() {
let p = ThreadPlacer::new();
if p.num_places() > 0 {
let place = p.assign_place(ProcBind::Master, 10);
assert!(place.is_some());
}
}
#[test]
fn test_thread_placer_assign_false() {
let p = ThreadPlacer::new();
let place = p.assign_place(ProcBind::False, 0);
assert!(place.is_none());
}
#[test]
fn test_thread_placer_place_procs() {
let p = ThreadPlacer::new();
if p.num_places() > 0 {
let procs = p.place_procs(0);
assert!(procs.is_some());
assert!(!procs.unwrap().is_empty());
}
}
#[test]
fn test_runtime_stats_default() {
let s = RuntimeStats::new();
assert_eq!(s.parallel_regions.load(Ordering::Relaxed), 0);
}
#[test]
fn test_runtime_stats_increment() {
let s = RuntimeStats::new();
s.parallel_regions.fetch_add(1, Ordering::SeqCst);
s.barriers.fetch_add(3, Ordering::SeqCst);
assert_eq!(s.parallel_regions.load(Ordering::Relaxed), 1);
assert_eq!(s.barriers.load(Ordering::Relaxed), 3);
}
#[test]
fn test_runtime_stats_reset() {
let s = RuntimeStats::new();
s.parallel_regions.store(10, Ordering::Release);
s.reset();
assert_eq!(s.parallel_regions.load(Ordering::Relaxed), 0);
}
#[test]
fn test_runtime_stats_report() {
let s = RuntimeStats::new();
let report = s.report();
assert!(report.contains("Parallel regions"));
assert!(report.contains("Tasks created"));
assert!(report.contains("Barriers"));
}
#[test]
fn test_omp_error_as_str() {
assert_eq!(OMPError::Success.as_str(), "success");
assert_eq!(OMPError::OutOfMemory.as_str(), "out of memory");
assert_eq!(OMPError::Internal.as_str(), "internal error");
}
#[test]
fn test_warning_no_crash() {
warning("test warning");
}
#[test]
fn test_x86_microarch_detect() {
let arch = X86MicroArch::detect();
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
assert_ne!(arch, X86MicroArch::Unknown);
}
#[test]
fn test_x86_microarch_wait_policy() {
assert_eq!(
X86MicroArch::AlderLake.preferred_wait_policy(),
WaitPolicy::Passive
);
assert_eq!(
X86MicroArch::Haswell.preferred_wait_policy(),
WaitPolicy::Active
);
}
#[test]
fn test_omp_get_version() {
assert!(omp_get_version() > 0);
}
#[test]
fn test_omp_get_api_version() {
assert!(omp_get_api_version() > 200000);
}
#[test]
fn test_omp_get_supported_active_levels() {
assert_eq!(
omp_get_supported_active_levels(),
OMP_MAX_ACTIVE_LEVELS as i32
);
}
#[test]
fn test_omp_get_device_num() {
assert_eq!(omp_get_device_num(), 0);
}
#[test]
fn test_omp_is_in_teams_region() {
assert_eq!(omp_is_in_teams_region(), 0);
}
#[test]
fn test_omp_is_in_target_region() {
assert_eq!(omp_is_in_target_region(), 0);
}
#[test]
fn test_omp_alloc_free() {
let p = omp_alloc(128, std::ptr::null());
assert!(!p.is_null());
omp_free(p, std::ptr::null());
}
#[test]
fn test_omp_alloc_zero() {
let p = omp_alloc(0, std::ptr::null());
assert!(p.is_null());
}
#[test]
fn test_omp_calloc() {
let p = omp_calloc(16, 8, std::ptr::null());
assert!(!p.is_null());
unsafe {
for i in 0..(16 * 8) {
assert_eq!(*p.add(i), 0);
}
}
}
#[test]
fn test_omp_calloc_zero_size() {
let p = omp_calloc(0, 8, std::ptr::null());
assert!(p.is_null());
}
#[test]
fn test_omp_realloc() {
let p = omp_alloc(64, std::ptr::null());
let p2 = omp_realloc(p, 128, std::ptr::null(), std::ptr::null());
if !p2.is_null() {
omp_free(p2, std::ptr::null());
}
}
#[test]
fn test_omp_allocator_handles_exist() {
let _ = &omp_default_mem_alloc;
let _ = &omp_large_cap_mem_alloc;
let _ = &omp_const_mem_alloc;
let _ = &omp_high_bw_mem_alloc;
let _ = &omp_low_lat_mem_alloc;
let _ = &omp_cgroup_mem_alloc;
let _ = &omp_pteam_mem_alloc;
let _ = &omp_thread_mem_alloc;
}
#[test]
fn test_kmpc_alloc_shared() {
let p = unsafe { __kmpc_alloc_shared(256) };
assert!(!p.is_null());
unsafe { __kmpc_free_shared(p, 256) };
}
#[test]
fn test_task_modify_flags() {
let mut task = TaskDescriptor::new(1, 0);
task.flags = 0;
let new_flags = unsafe {
__kmpc_omp_task_modify_flags(&mut task, TASK_FLAG_UNTIED | TASK_FLAG_FINAL, 0)
};
assert!(new_flags & TASK_FLAG_UNTIED != 0);
assert!(new_flags & TASK_FLAG_FINAL != 0);
}
#[test]
fn test_task_modify_flags_clear() {
let mut task = TaskDescriptor::new(1, 0);
task.flags = TASK_FLAG_UNTIED | TASK_FLAG_MERGABLE;
let new_flags = unsafe { __kmpc_omp_task_modify_flags(&mut task, 0, TASK_FLAG_UNTIED) };
assert!(new_flags & TASK_FLAG_UNTIED == 0);
assert!(new_flags & TASK_FLAG_MERGABLE != 0);
}
#[test]
fn test_task_modify_flags_null() {
let ret = unsafe { __kmpc_omp_task_modify_flags(std::ptr::null_mut(), 0, 0) };
assert_eq!(ret, -1);
}
#[test]
fn test_kmpc_for_static_next_returns_zero() {
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 0i32;
let mut st = 1i32;
let r = unsafe { __kmpc_for_static_next(&ident, 0, &mut last, &mut lo, &mut hi, &mut st) };
assert_eq!(r, 0);
}
#[test]
fn test_kmpc_dispatch_init_4_alias() {
let rt = X86OpenMP::new();
with_region(&rt, 1, || {
let ident = Ident::default();
unsafe {
__kmpc_dispatch_init_4(&ident, 0, schedule::SCHEDULE_DYNAMIC, 0, 99, 1, 5);
}
});
}
#[test]
fn test_kmpc_dispatch_init_8_alias() {
let rt = X86OpenMP::new();
with_region(&rt, 1, || {
let ident = Ident::default();
unsafe {
__kmpc_dispatch_init_8(&ident, 0, schedule::SCHEDULE_DYNAMIC, 0, 999, 1, 10);
}
});
}
#[test]
fn test_kmpc_fork_teams_single() {
init_test();
omp_set_num_threads(1);
let ident = Ident::default();
unsafe extern "C" fn mt(_g: i32, _b: i32) {}
unsafe {
__kmpc_fork_teams(&ident, 0, mt);
}
}
#[test]
fn test_for_static_init_4_simple() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i32;
let mut hi = 99i32;
let mut st = 1i32;
unsafe {
__kmpc_for_static_init_4_simple(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
}
#[test]
fn test_for_static_init_8_simple() {
init_test();
let ident = Ident::default();
let mut last = 0i32;
let mut lo = 0i64;
let mut hi = 99i64;
let mut st = 1i64;
unsafe {
__kmpc_for_static_init_8_simple(
&ident,
0,
schedule::SCHEDULE_STATIC,
&mut last,
&mut lo,
&mut hi,
&mut st,
1,
0,
);
}
}
#[test]
fn test_omp_control_tool() {
let ret = omp_control_tool(0, 0, std::ptr::null_mut());
assert_eq!(ret, -1);
}
#[test]
fn test_omp_get_ancestor_thread_num_valid() {
init_test();
let rt = X86OpenMP::new();
let r = Arc::new(ParallelRegionState::new(0, 2));
rt.push_parallel_region(r);
assert_eq!(omp_get_ancestor_thread_num(0), 0);
rt.pop_parallel_region();
}
#[test]
fn test_lock_double_destroy_safe() {
let mut lock: *mut OMPLock = std::ptr::null_mut();
omp_init_lock(&mut lock);
omp_destroy_lock(&mut lock);
omp_destroy_lock(&mut lock); }
#[test]
fn test_nest_lock_max_depth() {
let lock = OMPNestLock::new();
for _ in 0..OMP_MAX_NEST_LOCK_DEPTH + 10 {
lock.set(1);
}
assert!(lock.is_locked());
for _ in 0..OMP_MAX_NEST_LOCK_DEPTH {
lock.unset();
}
assert!(!lock.is_locked());
}
#[test]
fn test_taskloop_concurrent_chunks() {
let state = Arc::new(TaskLoopState::new(0, 99, 1, 10, 0));
let mut handles = Vec::new();
for _ in 0..4 {
let s = Arc::clone(&state);
handles.push(std::thread::spawn(move || {
let mut chunks = Vec::new();
while let Some((lo, hi)) = s.next_chunk() {
chunks.push((lo, hi));
}
chunks
}));
}
let mut all_chunks = Vec::new();
for h in handles {
all_chunks.extend(h.join().unwrap());
}
all_chunks.sort_by_key(|&(lo, _)| lo);
let mut expected = 0i64;
for (lo, hi) in all_chunks {
assert_eq!(lo, expected, "gap in chunk coverage");
expected = hi + 1;
}
assert_eq!(expected, 100);
}
#[test]
fn test_barrier_concurrent() {
let b = Arc::new(TeamBarrier::new(8));
let mut handles = Vec::new();
for i in 0..8 {
let barrier = Arc::clone(&b);
handles.push(std::thread::spawn(move || {
barrier.wait();
i
}));
}
for h in handles {
h.join().unwrap();
}
}
#[test]
fn test_single_only_one_executes() {
let single = Arc::new(SingleState::new());
let counter = Arc::new(AtomicU32::new(0));
let mut handles = Vec::new();
for _ in 0..8 {
let s = Arc::clone(&single);
let c = Arc::clone(&counter);
handles.push(std::thread::spawn(move || {
if s.try_acquire() {
c.fetch_add(1, Ordering::SeqCst);
}
}));
}
for h in handles {
h.join().unwrap();
}
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[test]
fn test_lock_multithread_contention() {
let lock = Arc::new(OMPLock::new());
let counter = Arc::new(AtomicU32::new(0));
let mut handles = Vec::new();
for _ in 0..4 {
let l = Arc::clone(&lock);
let c = Arc::clone(&counter);
handles.push(std::thread::spawn(move || {
for _ in 0..100 {
l.set(1);
let val = c.load(Ordering::SeqCst);
c.store(val + 1, Ordering::SeqCst);
l.unset();
}
}));
}
for h in handles {
h.join().unwrap();
}
assert_eq!(counter.load(Ordering::SeqCst), 400);
}
#[test]
fn test_reduction_table_register_unregister_cycle() {
let mut t = ReductionTable::new();
let initial_len = t.len();
t.register(
"test_cycle".into(),
ReductionTypeInfo {
type_size: 4,
op: ReductionOp::Custom,
init_fn: |_| {},
combine_fn: |_, _| {},
finalize_fn: None,
},
);
assert_eq!(t.len(), initial_len + 1);
t.unregister("test_cycle");
assert_eq!(t.len(), initial_len);
}
#[test]
fn test_static_bounds_full_coverage() {
let n_thr = 7u32;
let total = 103i64;
let chunk = 2i64;
let mut coverage = vec![false; total as usize];
for tid in 0..n_thr {
let (lo, hi, _) = compute_static_bounds(total, chunk, n_thr, tid, 0, 1, total - 1);
if lo <= hi {
for i in lo..=hi {
assert!(!coverage[i as usize], "overlap at {}", i);
coverage[i as usize] = true;
}
}
}
for (i, &covered) in coverage.iter().enumerate() {
assert!(covered, "iteration {} not covered", i);
}
}
#[test]
fn test_taskwait_executes_pending() {
init_test();
let counter = Arc::new(AtomicI32::new(0));
let q = TaskQueue::new();
unsafe extern "C" fn task_func(_g: i32, _t: *mut TaskDescriptor) {
}
let mut t = TaskDescriptor::new(1, 0);
t.task_fn = Some(task_func);
q.enqueue_task(Arc::new(t));
q.taskwait(0);
assert_eq!(q.ready_count(), 0);
}
#[test]
fn test_taskgroup_nested() {
let q = TaskQueue::new();
q.begin_taskgroup();
q.begin_taskgroup();
q.end_taskgroup(0);
q.end_taskgroup(0);
}
#[test]
fn test_omp_get_max_task_priority_value() {
assert!(omp_get_max_task_priority() >= 0);
}
#[test]
fn test_x86_simd_vector_length_non_x86() {
#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
assert_eq!(x86_simd_vector_length(), X86SimdWidth::SSE);
}
#[test]
fn test_simd_width_names() {
assert_eq!(X86SimdWidth::SSE.name(), "SSE (128-bit)");
assert_eq!(X86SimdWidth::AVX.name(), "AVX (256-bit)");
assert_eq!(X86SimdWidth::AVX512.name(), "AVX-512 (512-bit)");
}
#[test]
fn test_icv_stack_size_default() {
let icv = ICVStorage::default();
assert_eq!(icv.stack_size, OMP_DEFAULT_STACK_SIZE);
}
#[test]
fn test_icv_wait_policy_inherit() {
let mut parent = ICVStorage::default();
parent.wait_policy = WaitPolicy::Passive;
let child = parent.inherit();
assert_eq!(child.wait_policy, WaitPolicy::Passive);
}
#[test]
fn test_icv_proc_bind_inherit() {
let mut parent = ICVStorage::default();
parent.proc_bind = ProcBind::Close;
let child = parent.inherit();
assert_eq!(child.proc_bind, ProcBind::Close);
}
#[test]
fn test_thread_pool_wait_policy_roundtrip() {
let pool = ThreadPool::new();
assert_eq!(*pool.wait_policy.read().unwrap(), WaitPolicy::Active);
pool.set_wait_policy(WaitPolicy::Passive);
assert_eq!(*pool.wait_policy.read().unwrap(), WaitPolicy::Passive);
pool.set_wait_policy(WaitPolicy::Active);
assert_eq!(*pool.wait_policy.read().unwrap(), WaitPolicy::Active);
}
#[test]
fn test_critical_lock_table_len() {
let t = CriticalLockTable::new();
assert_eq!(t.len(), 0);
t.get_or_create("a");
assert_eq!(t.len(), 1);
t.get_or_create("b");
assert_eq!(t.len(), 2);
t.remove("a");
assert_eq!(t.len(), 1);
}
#[test]
fn test_task_queue_ready_waiting_counts() {
let q = TaskQueue::new();
assert_eq!(q.ready_count(), 0);
assert_eq!(q.waiting_count(), 0);
q.enqueue_task(Arc::new(TaskDescriptor::new(1, 0)));
assert_eq!(q.ready_count(), 1);
q.dequeue_task();
assert_eq!(q.ready_count(), 0);
}
#[test]
fn test_taskgroup_nested_depths() {
let q = TaskQueue::new();
q.begin_taskgroup(); q.begin_taskgroup(); q.end_taskgroup(0); q.end_taskgroup(0); }
#[test]
fn test_team_workshare_active_toggle() {
let tw = TeamWorkshare::new();
assert!(!tw.active.load(Ordering::Acquire));
tw.active.store(true, Ordering::Release);
assert!(tw.active.load(Ordering::Acquire));
tw.active.store(false, Ordering::Release);
assert!(!tw.active.load(Ordering::Acquire));
}
#[test]
fn test_region_cancelled_toggle() {
let r = ParallelRegionState::new(0, 2);
assert!(!r.cancelled.load(Ordering::Acquire));
r.cancelled.store(true, Ordering::Release);
assert!(r.cancelled.load(Ordering::Acquire));
}
#[test]
fn test_reduction_context_lock() {
let ctx = ReductionContext::new();
{
let _g = ctx.lock.lock().unwrap();
assert!(ctx.lock.try_lock().is_err());
}
assert!(ctx.lock.try_lock().is_ok());
}
#[test]
fn test_reduction_context_generation() {
let ctx = ReductionContext::new();
assert_eq!(ctx.generation.load(Ordering::Acquire), 0);
ctx.generation.fetch_add(1, Ordering::SeqCst);
assert_eq!(ctx.generation.load(Ordering::Acquire), 1);
}
#[test]
fn test_barrier_cancelled_flag() {
let b = TeamBarrier::new(1);
assert!(!b.is_cancelled());
b.cancel();
assert!(b.is_cancelled());
}
#[test]
fn test_x86ompp_is_initialized() {
let rt = X86OpenMP::new();
assert!(!rt.is_initialized());
rt.initialize();
assert!(rt.is_initialized());
}
#[test]
fn test_omp_get_num_procs_positive() {
assert!(omp_get_num_procs() >= 1);
}
#[test]
fn test_omp_get_wtick_range() {
let t = omp_get_wtick();
assert!(t > 0.0);
assert!(t < 1.0);
}
#[test]
fn test_omp_get_wtime_monotonic() {
let t1 = omp_get_wtime();
std::thread::sleep(Duration::from_millis(1));
let t2 = omp_get_wtime();
assert!(t2 > t1);
}
#[test]
fn test_kmpc_end_critical_no_begin() {
let ident = Ident::default();
let lock: usize = 0x1234;
unsafe {
__kmpc_end_critical(&ident, 0, lock as *mut ());
}
}
#[test]
fn test_kmpc_single_no_region() {
init_test();
let ident = Ident::default();
assert_eq!(unsafe { __kmpc_single(&ident, 0) }, 0);
}
#[test]
fn test_omp_lock_multiple_cycles() {
let lock = Arc::new(OMPLock::new());
for _ in 0..100 {
lock.set(0);
lock.unset();
}
assert!(!lock.is_locked());
}
#[test]
fn test_barrier_multiple_generations() {
let b = TeamBarrier::new(1);
for _ in 0..100 {
b.wait();
}
assert_eq!(b.generation.load(Ordering::Acquire), 100);
}
#[test]
fn test_get_team_size_invalid() {
let rt = X86OpenMP::new();
assert_eq!(rt.get_team_size(0), -1);
assert_eq!(rt.get_team_size(100), -1);
}
#[test]
fn test_get_ancestor_invalid_levels() {
let rt = X86OpenMP::new();
assert_eq!(rt.get_ancestor_thread_num(0), -1);
assert_eq!(rt.get_ancestor_thread_num(50), -1);
}
#[test]
fn test_task_descriptor_creation_timestamp() {
let before = Instant::now();
let t = TaskDescriptor::new(100, 0);
assert!(t.created_at >= before);
assert!(t.created_at <= Instant::now());
}
#[test]
fn test_schedule_modifiers_distinct() {
assert_ne!(
schedule::SCHEDULE_MONOTONIC,
schedule::SCHEDULE_NONMONOTONIC
);
assert!(schedule::is_monotonic(
schedule::SCHEDULE_STATIC | schedule::SCHEDULE_MONOTONIC
));
assert!(!schedule::is_monotonic(
schedule::SCHEDULE_STATIC | schedule::SCHEDULE_NONMONOTONIC
));
}
#[test]
fn test_icv_cancellation_inherit() {
let mut parent = ICVStorage::default();
parent.cancellation = true;
let child = parent.inherit();
assert!(child.cancellation);
}
#[test]
fn test_icv_default_device_inherit() {
let mut parent = ICVStorage::default();
parent.default_device = 3;
let child = parent.inherit();
assert_eq!(child.default_device, 3);
}
#[test]
fn test_runtime_places_after_init() {
let rt = X86OpenMP::new();
rt.initialize();
let places = rt.places.read().unwrap();
assert!(!places.is_empty());
}
#[test]
fn test_omp_get_version_range() {
let v = omp_get_version();
assert!(v >= 199910); }
#[test]
fn test_icv_default_schedule_chunk() {
let icv = ICVStorage::default();
assert_eq!(icv.default_schedule, schedule::SCHEDULE_STATIC);
assert_eq!(icv.default_chunk, 0);
}
#[test]
fn test_event_trace_default_disabled() {
let t = EventTrace::default();
assert!(!t.is_enabled());
assert!(t.is_empty());
}
#[test]
fn test_barrier_cancelled_state_after_cancel() {
let b = Arc::new(TeamBarrier::new(2));
let b2 = Arc::clone(&b);
let h = std::thread::spawn(move || b2.wait_cancellable());
std::thread::sleep(Duration::from_millis(5));
b.cancel();
let cancelled = h.join().unwrap();
assert!(cancelled);
assert!(b.is_cancelled());
}
#[test]
fn test_workshare_initialized_flag() {
let ws = WorkshareState::default();
assert!(!ws.initialized);
}
#[test]
fn test_dist_for_chunk_stride_default() {
let df = DistForState::default();
for i in 0..64 {
assert_eq!(df.chunk_stride[i], 1);
}
}
#[test]
fn test_taskloop_tasks_created() {
let s = TaskLoopState::new(0, 9, 1, 0, 5);
while s.next_chunk().is_some() {}
assert_eq!(s.tasks_created.load(Ordering::Relaxed), 5);
}
#[test]
fn test_omp_taskyield_no_tasks() {
init_test();
let ident = Ident::default();
unsafe {
__kmpc_omp_taskyield(&ident, 0);
}
}
#[test]
fn test_kmpc_end_single_no_region() {
init_test();
let ident = Ident::default();
unsafe {
__kmpc_end_single(&ident, 0);
}
}
#[test]
fn test_omp_error_debug_format() {
let e = OMPError::ThreadLimitExceeded;
assert_eq!(format!("{:?}", e), "ThreadLimitExceeded");
}
#[test]
fn test_simd_width_ord_complete() {
let mut widths = vec![X86SimdWidth::AVX512, X86SimdWidth::SSE, X86SimdWidth::AVX];
widths.sort();
assert_eq!(
widths,
vec![X86SimdWidth::SSE, X86SimdWidth::AVX, X86SimdWidth::AVX512,]
);
}
#[test]
fn test_runtime_stats_peak_threads() {
let s = RuntimeStats::new();
assert_eq!(s.peak_threads.load(Ordering::Relaxed), 0);
s.peak_threads.store(8, Ordering::Release);
assert_eq!(s.peak_threads.load(Ordering::Relaxed), 8);
}
#[test]
fn test_thread_placer_place_procs_out_of_range() {
let p = ThreadPlacer::new();
let procs = p.place_procs(999999);
assert!(procs.is_none());
}
#[test]
#[should_panic]
fn test_fatal_error_handler_set_and_invoke() {
set_fatal_error_handler(|_e, _m| panic!("custom handler"));
fatal_error(OMPError::Internal, "test fatal");
}
}