1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

//! Low-level utils
 
#![allow(unused)]

#[inline]
pub fn cpu() -> usize {
    std::thread::current().id().as_u64().get() as usize
}

#[cfg(target_arch = "x86")]
use std::arch::x86::{_mm_mfence, _mm_sfence, clflush};

#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::{_mm_clflush, _mm_mfence, _mm_sfence};

/// Synchronize caches and memories and acts like a write barrier
#[inline]
pub fn msync<T: ?Sized>(ptr: &T, len: usize) {
    #[cfg(not(feature = "no_persist"))]
    {
        #[cfg(not(feature = "use_msync"))]
        clflush(ptr, len);

        #[cfg(feature = "use_msync")]
        unsafe {
            let off = ptr as usize;
            let end = off + len;
            let off = (off >> 12) << 12;
            let len = end - off;
            let ptr = off as *const u8;
            if libc::msync(
                ptr as *mut libc::c_void,
                len,
                libc::MS_SYNC | libc::MS_INVALIDATE,
            ) != 0
            {
                panic!("msync failed");
            }
        }
    }
}

/// Synchronize caches and memories and acts like a write barrier
#[inline]
pub fn msync_obj<T: ?Sized>(obj: &T) {
    #[cfg(not(feature = "no_persist"))]
    {
        #[cfg(not(feature = "use_msync"))]
        clflush_obj(obj);

        #[cfg(feature = "use_msync")]
        {
            let ptr = obj as *const T as *const u8 as *mut u8;
            let len = std::mem::size_of_val(obj);
            msync(ptr, len);
        }
    }
}

/// Flushes cache line back to memory
#[inline]
pub fn clflush<T: ?Sized>(ptr: &T, len: usize) {
    #[cfg(not(feature = "no_persist"))]
    {
        let ptr = ptr as *const _ as *const u8 as *mut u8;
        let mut start = ptr as usize;
        let end = start + len;

        #[cfg(feature = "display_all_flushes")]
        println!("flush {:x} ({})", start, len);

        while start < end {
            unsafe {
                #[cfg(not(any(feature = "use_clflushopt", feature = "use_clwb")))]
                {
                    llvm_asm!("clflush ($0)" :: "r"(start as *const u8));
                }
                #[cfg(all(feature = "use_clflushopt", not(feature = "use_clwb")))]
                {
                    llvm_asm!("clflushopt ($0)" :: "r"(start as *const u8));
                    #[cfg(feature = "use_sfence")]
                    {
                        llvm_asm!("sfence");
                    }
                }
                #[cfg(all(feature = "use_clwb", not(feature = "use_clflushopt")))]
                {
                    llvm_asm!("clwb ($0)" :: "r"(start as *const u8));
                    #[cfg(feature = "use_sfence")]
                    {
                        llvm_asm!("sfence");
                    }
                }
                #[cfg(all(feature = "use_clwb", feature = "use_clflushopt"))]
                {
                    compile_error!("Please Select only one from clflushopt and clwb")
                }
            }
            start += 64;
        }
    }
}

/// Flushes cache lines of a whole object back to memory
#[inline]
pub fn clflush_obj<T: ?Sized>(obj: &T) {
    #[cfg(not(feature = "no_persist"))]
    {
        let len = std::mem::size_of_val(obj);
        clflush(obj, len)
    }
}

/// Store fence
#[inline]
pub fn sfence() {
    unsafe {
        _mm_sfence();
    }
}

/// Memory fence
#[inline]
pub fn mfence() {
    unsafe {
        _mm_mfence();
    }
}