crndm 0.1.0

Persistent Programming Library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

//! Low-level utils
 
#![allow(unused)]

#[cfg(target_arch = "x86")]
use std::arch::x86::{_mm_mfence, _mm_sfence, clflush};

#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::{_mm_clflush, _mm_mfence, _mm_sfence};

/// Synchronize caches and memories and acts like a write barrier
#[inline]
pub fn msync<T: ?Sized>(ptr: &T, len: usize) {
    #[cfg(not(feature = "no_persist"))]
    {
        #[cfg(not(feature = "use_msync"))]
        clflush(ptr, len);

        #[cfg(feature = "use_msync")]
        unsafe {
            let off = ptr as usize;
            let end = off + len;
            let off = (off >> 12) << 12;
            let len = end - off;
            let ptr = off as *const u8;
            if libc::msync(
                ptr as *mut libc::c_void,
                len,
                libc::MS_SYNC | libc::MS_INVALIDATE,
            ) != 0
            {
                panic!("msync failed");
            }
        }
    }
}

/// Synchronize caches and memories and acts like a write barrier
#[inline]
pub fn msync_obj<T: ?Sized>(obj: &T) {
    #[cfg(not(feature = "no_persist"))]
    {
        #[cfg(not(feature = "use_msync"))]
        clflush_obj(obj);

        #[cfg(feature = "use_msync")]
        {
            let ptr = obj as *const T as *const u8 as *mut u8;
            let len = std::mem::size_of_val(obj);
            msync(ptr, len);
        }
    }
}

/// Flushes cache line back to memory
#[inline]
pub fn clflush<T: ?Sized>(ptr: &T, len: usize) {
    #[cfg(not(feature = "no_persist"))]
    {
        let ptr = ptr as *const _ as *const u8 as *mut u8;
        let mut start = ptr as usize;
        let end = start + len;

        #[cfg(feature = "display_all_flushes")]
        println!("flush {:x} ({})", start, len);

        while start < end {
            unsafe {
                #[cfg(not(any(feature = "use_clflushopt", feature = "use_clwb")))]
                {
                    llvm_asm!("clflush ($0)" :: "r"(start as *const u8));
                }
                #[cfg(all(feature = "use_clflushopt", not(feature = "use_clwb")))]
                {
                    llvm_asm!("clflushopt ($0)" :: "r"(start as *const u8));
                    #[cfg(feature = "use_sfence")]
                    {
                        llvm_asm!("sfence");
                    }
                }
                #[cfg(all(feature = "use_clwb", not(feature = "use_clflushopt")))]
                {
                    llvm_asm!("clwb ($0)" :: "r"(start as *const u8));
                    #[cfg(feature = "use_sfence")]
                    {
                        llvm_asm!("sfence");
                    }
                }
                #[cfg(all(feature = "use_clwb", feature = "use_clflushopt"))]
                {
                    compile_error!("Please Select only one from clflushopt and clwb")
                }
            }
            start += 64;
        }
    }
}

/// Flushes cache lines of a whole object back to memory
#[inline]
pub fn clflush_obj<T: ?Sized>(obj: &T) {
    #[cfg(not(feature = "no_persist"))]
    {
        let len = std::mem::size_of_val(obj);
        clflush(obj, len)
    }
}

/// Store fence
#[inline]
pub fn sfence() {
    unsafe {
        _mm_sfence();
    }
}

/// Memory fence
#[inline]
pub fn mfence() {
    unsafe {
        _mm_mfence();
    }
}