use std::any::Any;
use std::ops::{Add, Deref, DerefMut, Sub};
use std::panic::Location;
use std::sync::{Arc, Mutex, MutexGuard};
use crate::ctx::ThreadCtx;
use crate::diag::{AccessKind, Diagnostic, MemSpace};
use crate::shadow::Cell;
use crate::sync::Ordering;
struct Instrumented<T> {
data: Vec<T>,
shadow: Vec<Cell>,
launch_id: u64,
}
struct Buffer<'a, T, S, R> {
mem: &'a Mutex<Instrumented<T>>,
space: S,
reset: R,
}
fn checked_access<T, S, R>(
buf: Buffer<'_, T, S, R>,
ctx: &ThreadCtx<'_>,
index: usize,
kind: AccessKind,
location: &'static Location<'static>,
ordering: Option<Ordering>,
op: impl FnOnce(&mut T) -> T,
) -> T
where
T: Copy,
S: Fn() -> MemSpace,
R: Fn(&mut Instrumented<T>),
{
let Buffer { mem, space, reset } = buf;
ctx.schedule_point();
let mut m = mem.lock().unwrap();
if m.launch_id != ctx.launch_id() {
reset(&mut m);
m.launch_id = ctx.launch_id();
}
let len = m.data.len();
if index >= len {
drop(m);
let (access, _) = ctx.access(kind, location);
ctx.trap(Diagnostic::OutOfBounds {
space: space(),
index,
len,
access,
});
}
if let Some(ordering) = ordering {
let published = m.shadow[index].release.clone();
ctx.acquire_from(&published, ordering);
}
let (acc, now) = ctx.access(kind, location);
let cell = &mut m.shadow[index];
let was_init = cell.init;
let conflict = match kind {
AccessKind::Read => cell.on_read(acc, &now),
AccessKind::Write | AccessKind::Atomic => cell.on_write(acc, &now),
};
let value = op(&mut m.data[index]);
if let Some(ordering) = ordering {
ctx.release_into(&mut m.shadow[index].release, ordering);
}
drop(m);
if kind != AccessKind::Write && !was_init {
ctx.report(Diagnostic::UninitRead {
space: space(),
index,
access: acc,
});
}
if let Some(first) = conflict {
ctx.report(Diagnostic::DataRace {
space: space(),
index,
first,
second: acc,
});
}
value
}
pub struct GlobalBuf<T> {
name: Arc<str>,
mem: Arc<Mutex<Instrumented<T>>>,
}
impl<T> Clone for GlobalBuf<T> {
fn clone(&self) -> Self {
GlobalBuf {
name: self.name.clone(),
mem: self.mem.clone(),
}
}
}
impl<T: Copy + Send + 'static> GlobalBuf<T> {
pub fn new(name: &str, data: Vec<T>) -> Self {
let shadow = vec![Cell::initialised(); data.len()];
GlobalBuf {
name: name.into(),
mem: Arc::new(Mutex::new(Instrumented {
data,
shadow,
launch_id: 0,
})),
}
}
pub fn uninit(name: &str, len: usize) -> Self
where
T: Default,
{
GlobalBuf {
name: name.into(),
mem: Arc::new(Mutex::new(Instrumented {
data: (0..len).map(|_| T::default()).collect(),
shadow: vec![Cell::default(); len],
launch_id: 0,
})),
}
}
pub fn len(&self) -> usize {
self.mem.lock().unwrap().data.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn to_vec(&self) -> Vec<T> {
self.mem.lock().unwrap().data.clone()
}
fn space(&self) -> MemSpace {
MemSpace::Global {
buffer: self.name.clone(),
}
}
fn reset(m: &mut Instrumented<T>) {
for c in &mut m.shadow {
let init = c.init;
*c = Cell::default();
c.init = init;
}
}
#[track_caller]
pub fn read(&self, ctx: &ThreadCtx<'_>, index: usize) -> T {
checked_access(
Buffer {
mem: &self.mem,
space: || self.space(),
reset: Self::reset,
},
ctx,
index,
AccessKind::Read,
Location::caller(),
None,
|v| *v,
)
}
#[track_caller]
pub fn write(&self, ctx: &ThreadCtx<'_>, index: usize, value: T) {
checked_access(
Buffer {
mem: &self.mem,
space: || self.space(),
reset: Self::reset,
},
ctx,
index,
AccessKind::Write,
Location::caller(),
None,
|v| {
*v = value;
value
},
);
}
}
impl<T: Copy + Send + 'static> GlobalBuf<T> {
#[track_caller]
pub fn atomic_load(&self, ctx: &ThreadCtx<'_>, index: usize, ordering: Ordering) -> T {
self.atomic_load_at(ctx, index, ordering, Location::caller())
}
pub(crate) fn atomic_load_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| *v)
}
#[track_caller]
pub fn atomic_store(&self, ctx: &ThreadCtx<'_>, index: usize, value: T, ordering: Ordering) {
self.atomic_store_at(ctx, index, value, ordering, Location::caller());
}
pub(crate) fn atomic_store_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) {
self.atomic_rmw(ctx, index, ordering, at, |v| {
*v = value;
value
});
}
}
impl<T: Copy + Send + 'static> GlobalBuf<T> {
pub(crate) fn atomic_rmw(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
ordering: Ordering,
at: &'static Location<'static>,
update: impl FnOnce(&mut T) -> T,
) -> T {
checked_access(
Buffer {
mem: &self.mem,
space: || self.space(),
reset: Self::reset,
},
ctx,
index,
AccessKind::Atomic,
at,
Some(ordering),
update,
)
}
#[track_caller]
pub fn atomic_swap(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
) -> T {
self.atomic_swap_at(ctx, index, value, ordering, Location::caller())
}
pub(crate) fn atomic_swap_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| std::mem::replace(v, value))
}
}
impl<T: Copy + Send + PartialEq + 'static> GlobalBuf<T> {
#[track_caller]
pub fn atomic_compare_exchange(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
current: T,
new: T,
ordering: Ordering,
) -> Result<T, T> {
self.atomic_compare_exchange_at(ctx, index, current, new, ordering, Location::caller())
}
pub(crate) fn atomic_compare_exchange_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
current: T,
new: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> Result<T, T> {
let mut exchanged = false;
let previous = self.atomic_rmw(ctx, index, ordering, at, |v| {
let previous = *v;
if previous == current {
*v = new;
exchanged = true;
}
previous
});
if exchanged {
Ok(previous)
} else {
Err(previous)
}
}
}
impl<T: Copy + Send + Ord + 'static> GlobalBuf<T> {
#[track_caller]
pub fn atomic_min(&self, ctx: &ThreadCtx<'_>, index: usize, value: T, ordering: Ordering) -> T {
self.atomic_min_at(ctx, index, value, ordering, Location::caller())
}
pub(crate) fn atomic_min_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| {
let previous = *v;
if value < previous {
*v = value;
}
previous
})
}
#[track_caller]
pub fn atomic_max(&self, ctx: &ThreadCtx<'_>, index: usize, value: T, ordering: Ordering) -> T {
self.atomic_max_at(ctx, index, value, ordering, Location::caller())
}
pub(crate) fn atomic_max_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| {
let previous = *v;
if value > previous {
*v = value;
}
previous
})
}
}
impl<T: Copy + Send + Sub<Output = T> + 'static> GlobalBuf<T> {
#[track_caller]
pub fn atomic_sub(&self, ctx: &ThreadCtx<'_>, index: usize, value: T, ordering: Ordering) -> T {
self.atomic_sub_at(ctx, index, value, ordering, Location::caller())
}
pub(crate) fn atomic_sub_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| {
let previous = *v;
*v = previous - value;
previous
})
}
}
impl<T: Copy + Send + Add<Output = T> + 'static> GlobalBuf<T> {
#[track_caller]
pub fn atomic_add(&self, ctx: &ThreadCtx<'_>, index: usize, value: T, ordering: Ordering) -> T {
self.atomic_add_at(ctx, index, value, ordering, Location::caller())
}
pub(crate) fn atomic_add_at(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
value: T,
ordering: Ordering,
at: &'static Location<'static>,
) -> T {
self.atomic_rmw(ctx, index, ordering, at, |v| {
let previous = *v;
*v = previous + value;
previous
})
}
}
pub struct ElemMut<'a, T> {
guard: MutexGuard<'a, Instrumented<T>>,
index: usize,
}
impl<T> Deref for ElemMut<'_, T> {
type Target = T;
fn deref(&self) -> &T {
&self.guard.data[self.index]
}
}
impl<T> DerefMut for ElemMut<'_, T> {
fn deref_mut(&mut self) -> &mut T {
&mut self.guard.data[self.index]
}
}
impl<T: Copy + Send + 'static> GlobalBuf<T> {
pub(crate) fn elem_mut(
&self,
ctx: &ThreadCtx<'_>,
index: usize,
checked: bool,
location: &'static Location<'static>,
) -> Option<ElemMut<'_, T>> {
ctx.schedule_point();
let (acc, now) = ctx.access(AccessKind::Write, location);
let mut m = self.mem.lock().unwrap();
if m.launch_id != ctx.launch_id() {
Self::reset(&mut m);
m.launch_id = ctx.launch_id();
}
let len = m.data.len();
if index >= len {
drop(m);
if checked {
return None;
}
ctx.trap(Diagnostic::OutOfBounds {
space: self.space(),
index,
len,
access: acc,
});
}
if let Some(first) = m.shadow[index].on_write(acc, &now) {
ctx.report(Diagnostic::DataRace {
space: self.space(),
index,
first,
second: acc,
});
}
Some(ElemMut { guard: m, index })
}
}
pub(crate) struct SharedInner<T> {
block: u32,
name: &'static str,
mem: Mutex<Instrumented<T>>,
}
pub struct SharedArray<T> {
inner: Arc<SharedInner<T>>,
}
impl<T> Clone for SharedArray<T> {
fn clone(&self) -> Self {
SharedArray {
inner: self.inner.clone(),
}
}
}
impl<T: Copy + Default + Send + 'static> SharedArray<T> {
pub(crate) fn new_inner(block: u32, name: &'static str, len: usize) -> SharedInner<T> {
SharedInner {
block,
name,
mem: Mutex::new(Instrumented {
data: vec![T::default(); len],
shadow: vec![Cell::default(); len],
launch_id: 0,
}),
}
}
pub(crate) fn from_any(a: Arc<dyn Any + Send + Sync>) -> Option<Self> {
a.downcast::<SharedInner<T>>()
.ok()
.map(|inner| SharedArray { inner })
}
pub fn len(&self) -> usize {
self.inner.mem.lock().unwrap().data.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
fn space(&self) -> MemSpace {
MemSpace::Shared {
block: self.inner.block,
name: self.inner.name,
}
}
fn reset(m: &mut Instrumented<T>) {
for c in &mut m.shadow {
*c = Cell::default();
}
}
#[track_caller]
pub fn read(&self, ctx: &ThreadCtx<'_>, index: usize) -> T {
checked_access(
Buffer {
mem: &self.inner.mem,
space: || self.space(),
reset: Self::reset,
},
ctx,
index,
AccessKind::Read,
Location::caller(),
None,
|v| *v,
)
}
#[track_caller]
pub fn write(&self, ctx: &ThreadCtx<'_>, index: usize, value: T) {
checked_access(
Buffer {
mem: &self.inner.mem,
space: || self.space(),
reset: Self::reset,
},
ctx,
index,
AccessKind::Write,
Location::caller(),
None,
|v| {
*v = value;
value
},
);
}
}