#![allow(unused_imports)]
use super::atomic::*;
use super::invariant::*;
use super::modes::*;
use super::prelude::*;
verus! {
pub trait AtomicInvariantPredicate<K, V, G> {
spec fn atomic_inv(k: K, v: V, g: G) -> bool;
}
} macro_rules! declare_atomic_type {
($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => {
verus!{
pub struct $atomic_pred_ty<Pred> { p: Pred }
impl<K, G, Pred> InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty<Pred>
where Pred: AtomicInvariantPredicate<K, $value_ty, G>
{
open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool {
let (k, loc) = k_loc;
let (perm, g) = perm_g;
perm.view().patomic == loc
&& Pred::atomic_inv(k, perm.view().value, g)
}
}
#[doc = concat!(
"Sequentially-consistent atomic memory location storing a `",
stringify!($value_ty),
"` and associated ghost state."
)]
pub struct $at_ident<K, G, Pred>
{
#[doc(hidden)]
pub patomic: $patomic_ty,
#[doc(hidden)]
pub atomic_inv: Tracked<AtomicInvariant<(K, int), ($perm_ty, G), $atomic_pred_ty<Pred>>>,
}
impl<K, G, Pred> $at_ident<K, G, Pred>
where Pred: AtomicInvariantPredicate<K, $value_ty, G>
{
pub open spec fn well_formed(&self) -> bool {
self.atomic_inv@.constant().1 == self.patomic.id()
}
pub open spec fn constant(&self) -> K {
self.atomic_inv@.constant().0
}
#[inline(always)]
pub const fn new(Ghost(k): Ghost<K>, u: $value_ty, Tracked(g): Tracked<G>) -> (t: Self)
requires Pred::atomic_inv(k, u, g),
ensures t.well_formed() && t.constant() == k,
{
let (patomic, Tracked(perm)) = $patomic_ty::new(u);
let tracked pair = (perm, g);
assert(Pred::atomic_inv(k, u, g));
assert(perm.view().patomic == patomic.id());
let tracked atomic_inv = AtomicInvariant::new(
(k, patomic.id()), pair, 0);
$at_ident {
patomic,
atomic_inv: Tracked(atomic_inv),
}
}
#[inline(always)]
pub fn load(&self) -> $value_ty
requires self.well_formed(),
{
atomic_with_ghost!(self => load(); g => { })
}
#[inline(always)]
pub fn into_inner(self) -> (res: ($value_ty, Tracked<G>))
requires self.well_formed(),
ensures Pred::atomic_inv(self.constant(), res.0, res.1@),
{
let Self { patomic, atomic_inv: Tracked(atomic_inv) } = self;
let tracked (perm, g) = atomic_inv.into_inner();
let v = patomic.into_inner(Tracked(perm));
(v, Tracked(g))
}
}
}
};
}
macro_rules! declare_atomic_type_generic {
($at_ident:ident, $patomic_ty:ident, $perm_ty:ty, $value_ty: ty, $atomic_pred_ty: ident) => {
verus!{
pub struct $atomic_pred_ty<T, Pred> { t: T, p: Pred }
impl<T, K, G, Pred> InvariantPredicate<(K, int), ($perm_ty, G)> for $atomic_pred_ty<T, Pred>
where Pred: AtomicInvariantPredicate<K, $value_ty, G>
{
open spec fn inv(k_loc: (K, int), perm_g: ($perm_ty, G)) -> bool {
let (k, loc) = k_loc;
let (perm, g) = perm_g;
perm.view().patomic == loc
&& Pred::atomic_inv(k, perm.view().value, g)
}
}
#[doc = concat!(
"Sequentially-consistent atomic memory location storing a `",
stringify!($value_ty),
"` and associated ghost state."
)]
pub struct $at_ident<T, K, G, Pred>
{
#[doc(hidden)]
pub patomic: $patomic_ty<T>,
#[doc(hidden)]
pub atomic_inv: Tracked<AtomicInvariant<(K, int), ($perm_ty, G), $atomic_pred_ty<T, Pred>>>,
}
impl<T, K, G, Pred> $at_ident<T, K, G, Pred>
where Pred: AtomicInvariantPredicate<K, $value_ty, G>
{
pub open spec fn well_formed(&self) -> bool {
self.atomic_inv@.constant().1 == self.patomic.id()
}
pub open spec fn constant(&self) -> K {
self.atomic_inv@.constant().0
}
#[inline(always)]
pub const fn new(Ghost(k): Ghost<K>, u: $value_ty, Tracked(g): Tracked<G>) -> (t: Self)
requires Pred::atomic_inv(k, u, g),
ensures t.well_formed() && t.constant() == k,
{
let (patomic, Tracked(perm)) = $patomic_ty::<T>::new(u);
let tracked pair = (perm, g);
let tracked atomic_inv = AtomicInvariant::new(
(k, patomic.id()), pair, 0);
$at_ident {
patomic,
atomic_inv: Tracked(atomic_inv),
}
}
#[inline(always)]
pub fn load(&self) -> $value_ty
requires self.well_formed(),
{
atomic_with_ghost!(self => load(); g => { })
}
#[inline(always)]
pub fn into_inner(self) -> (res: ($value_ty, Tracked<G>))
requires self.well_formed(),
ensures Pred::atomic_inv(self.constant(), res.0, res.1@),
{
let Self { patomic, atomic_inv: Tracked(atomic_inv) } = self;
let tracked (perm, g) = atomic_inv.into_inner();
let v = patomic.into_inner(Tracked(perm));
(v, Tracked(g))
}
}
}
};
}
#[cfg(target_has_atomic = "64")]
declare_atomic_type!(AtomicU64, PAtomicU64, PermissionU64, u64, AtomicPredU64);
declare_atomic_type!(AtomicU32, PAtomicU32, PermissionU32, u32, AtomicPredU32);
declare_atomic_type!(AtomicU16, PAtomicU16, PermissionU16, u16, AtomicPredU16);
declare_atomic_type!(AtomicU8, PAtomicU8, PermissionU8, u8, AtomicPredU8);
declare_atomic_type!(AtomicUsize, PAtomicUsize, PermissionUsize, usize, AtomicPredUsize);
#[cfg(target_has_atomic = "64")]
declare_atomic_type!(AtomicI64, PAtomicI64, PermissionI64, i64, AtomicPredI64);
declare_atomic_type!(AtomicI32, PAtomicI32, PermissionI32, i32, AtomicPredI32);
declare_atomic_type!(AtomicI16, PAtomicI16, PermissionI16, i16, AtomicPredI16);
declare_atomic_type!(AtomicI8, PAtomicI8, PermissionI8, i8, AtomicPredI8);
declare_atomic_type!(AtomicIsize, PAtomicIsize, PermissionIsize, isize, AtomicPredIsize);
declare_atomic_type!(AtomicBool, PAtomicBool, PermissionBool, bool, AtomicPredBool);
declare_atomic_type_generic!(AtomicPtr, PAtomicPtr, PermissionPtr<T>, *mut T, AtomicPredPtr);
#[macro_export]
macro_rules! atomic_with_ghost {
($($tokens:tt)*) => {
::verus_builtin_macros::atomic_with_ghost_helper!(
$crate::vstd::atomic_ghost::atomic_with_ghost_inner,
$($tokens)*)
}
}
pub use atomic_with_ghost;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_inner {
(load, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_load!($e, $prev, $next, $ret, $g, $b)
};
(store, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_store!(
$e, $operand, $prev, $next, $ret, $g, $b
)
};
(swap, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
swap, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_or, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_or, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_and, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_and, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_xor, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_xor, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_nand, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_nand, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_max, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_max, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_min, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_min, $e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_add_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_add_wrapping,
$e,
$operand,
$prev,
$next,
$ret,
$g,
$b
)
};
(fetch_sub_wrapping, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_1_operand!(
fetch_sub_wrapping,
$e,
$operand,
$prev,
$next,
$ret,
$g,
$b
)
};
(fetch_add, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_add!(
$e, $operand, $prev, $next, $ret, $g, $b
)
};
(fetch_sub, $e:expr, ($operand:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_fetch_sub!(
$e, $operand, $prev, $next, $ret, $g, $b
)
};
(compare_exchange, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!(
compare_exchange,
$e,
$operand1,
$operand2,
$prev,
$next,
$ret,
$g,
$b
)
};
(compare_exchange_weak, $e:expr, ($operand1:expr, $operand2:expr), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_update_with_2_operand!(
compare_exchange_weak,
$e,
$operand1,
$operand2,
$prev,
$next,
$ret,
$g,
$b
)
};
(no_op, $e:expr, (), $prev:pat, $next:pat, $ret:pat, $g:ident, $b:block) => {
$crate::vstd::atomic_ghost::atomic_with_ghost_no_op!($e, $prev, $next, $ret, $g, $b)
};
}
pub use atomic_with_ghost_inner;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_store {
($e:expr, $operand:expr, $prev:pat, $next:pat, $res:pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let atomic = &($e);
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (mut perm, mut $g) = pair;
let ghost $prev = perm.view().value;
atomic.patomic.store(Tracked(&mut perm), $operand);
let ghost $next = perm.view().value;
let ghost $res = ();
proof { $b }
proof { pair = (perm, $g); }
});
} }
};
}
pub use atomic_with_ghost_store;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_load {
($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let result;
let atomic = &($e);
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (perm, mut $g) = pair;
result = atomic.patomic.load(Tracked(&perm));
let ghost $res = result;
let ghost $prev = result;
let ghost $next = result;
proof { $b }
proof { pair = (perm, $g); }
});
result
} }
};
}
pub use atomic_with_ghost_load;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_no_op {
($e:expr, $prev:pat, $next: pat, $res: pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let atomic = &($e);
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (perm, mut $g) = pair;
let ghost result = perm.view().value;
let ghost $res = result;
let ghost $prev = result;
let ghost $next = result;
proof { $b }
proof { pair = (perm, $g); }
});
} }
};
}
pub use atomic_with_ghost_no_op;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_update_with_1_operand {
($name:ident, $e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let result;
let atomic = &($e);
let operand = $operand;
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (mut perm, mut $g) = pair;
let ghost $prev = perm.view().value;
result = atomic.patomic.$name(Tracked(&mut perm), operand);
let ghost $res = result;
let ghost $next = perm.view().value;
proof { $b }
proof { pair = (perm, $g); }
});
result
} }
};
}
pub use atomic_with_ghost_update_with_1_operand;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_update_with_2_operand {
($name:ident, $e:expr, $operand1:expr, $operand2:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let result;
let atomic = &($e);
let operand1 = $operand1;
let operand2 = $operand2;
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (mut perm, mut $g) = pair;
let ghost $prev = perm.view().value;
result = atomic.patomic.$name(Tracked(&mut perm), operand1, operand2);
let ghost $res = result;
let ghost $next = perm.view().value;
proof { $b }
proof { pair = (perm, $g); }
});
result
} }
};
}
pub use atomic_with_ghost_update_with_2_operand;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_update_fetch_add {
($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => {
(::verus_builtin_macros::verus_exec_expr!( {
let result;
let atomic = &($e);
let operand = $operand;
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (mut perm, mut $g) = pair;
proof {
let $prev = perm.view().value as int;
let $res = perm.view().value as int;
let $next = perm.view().value as int + (operand as int);
{ $b }
}
result = atomic.patomic.fetch_add(Tracked(&mut perm), operand);
proof { pair = (perm, $g); }
});
result
} ))
}
}
pub use atomic_with_ghost_update_fetch_add;
#[doc(hidden)]
#[macro_export]
macro_rules! atomic_with_ghost_update_fetch_sub {
($e:expr, $operand:expr, $prev:pat, $next:pat, $res: pat, $g:ident, $b:block) => {
::verus_builtin_macros::verus_exec_expr! { {
let result;
let atomic = &($e);
let operand = $operand;
$crate::vstd::invariant::open_atomic_invariant!(atomic.atomic_inv.borrow() => pair => {
#[allow(unused_mut)]
let tracked (mut perm, mut $g) = pair;
proof {
let $prev = perm.view().value as int;
let $res = perm.view().value as int;
let $next = perm.view().value as int - (operand as int);
{ $b }
}
result = atomic.patomic.fetch_sub(Tracked(&mut perm), operand);
proof { pair = (perm, $g); }
});
result
} }
};
}
pub use atomic_with_ghost_update_fetch_sub;