1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#![allow(unused)]
#[inline(always)]
pub fn cpu() -> usize {
std::thread::current().id().as_u64().get() as usize
}
#[cfg(target_arch = "x86")]
use std::arch::x86::{_mm_mfence, _mm_sfence, clflush};
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::{_mm_clflush, _mm_mfence, _mm_sfence};
#[inline(always)]
pub fn persist<T: ?Sized>(ptr: &T, len: usize, fence: bool) {
#[cfg(feature = "stat_perf")]
let _perf = crate::stat::Measure::<crate::default::BuddyAlloc>::Sync(std::time::Instant::now());
#[cfg(not(feature = "no_persist"))]
{
#[cfg(not(feature = "use_msync"))]
clflush(ptr, len, fence);
#[cfg(feature = "use_msync")]
unsafe {
let off = ptr as *const T as *const u8 as usize;
let end = off + len;
let off = (off >> 12) << 12;
let len = end - off;
let ptr = off as *const u8;
if libc::persist(
ptr as *mut libc::c_void,
len,
libc::MS_SYNC | libc::MS_INVALIDATE,
) != 0
{
panic!("persist failed");
}
}
}
}
#[inline(always)]
pub fn persist_obj<T: ?Sized>(obj: &T, fence: bool) {
#[cfg(feature = "stat_perf")]
let _perf = crate::stat::Measure::<crate::default::BuddyAlloc>::Sync(std::time::Instant::now());
#[cfg(not(feature = "no_persist"))]
{
#[cfg(not(feature = "use_msync"))]
clflush_obj(obj, fence);
#[cfg(feature = "use_msync")]
{
persist(obj, std::mem::size_of_val(obj));
}
}
}
#[inline(always)]
pub fn clflush<T: ?Sized>(ptr: &T, len: usize, fence: bool) {
#[cfg(not(feature = "no_persist"))]
{
let ptr = ptr as *const _ as *const u8 as *mut u8;
let mut start = ptr as usize;
start = (start >> 9) << 9;
let end = start + len;
#[cfg(feature = "stat_print_flushes")]
println!("flush {:x} ({})", start, len);
while start < end {
unsafe {
#[cfg(not(any(feature = "use_clflushopt", feature = "use_clwb")))]
{
llvm_asm!("clflush ($0)" :: "r"(start as *const u8));
}
#[cfg(all(feature = "use_clflushopt", not(feature = "use_clwb")))]
{
llvm_asm!("clflushopt ($0)" :: "r"(start as *const u8));
}
#[cfg(all(feature = "use_clwb", not(feature = "use_clflushopt")))]
{
llvm_asm!("clwb ($0)" :: "r"(start as *const u8));
}
#[cfg(all(feature = "use_clwb", feature = "use_clflushopt"))]
{
compile_error!("Please Select only one from clflushopt and clwb")
}
}
start += 64;
}
}
}
#[inline(always)]
pub fn clflush_obj<T: ?Sized>(obj: &T, fence: bool) {
#[cfg(not(feature = "no_persist"))]
{
let len = std::mem::size_of_val(obj);
clflush(obj, len, fence)
}
}
#[inline(always)]
pub fn sfence() {
#[cfg(any(feature = "use_clwb", feature = "use_clflushopt"))] unsafe {
_mm_sfence();
}
}
#[inline]
pub fn mfence() {
unsafe {
_mm_mfence();
}
}