ulock_sys/
weak.rs

1//! `feature = "experimental-weak"` Weak linking emulation. See crate docs for info.
2use super::*;
3
4/// A dynamically loaded pair of function pointers implementing the API.
5///
6/// This can be returned by `ULockApi::get()`. Note that unless you can cache
7/// the API reference somewhere easily, it's a bit more efficient to call
8/// [`weak::ulock_wait`](ulock_wait) and/or [`weak::ulock_wake`](ulock_wake)
9/// directly.
10#[derive(Clone, Copy, PartialEq)]
11#[repr(C)]
12pub struct ULockApi {
13    /// Function pointer to [`__ulock_wait`].
14    pub ulock_wait: ULockWaitFn,
15    /// Function pointer to [`__ulock_wake`].
16    pub ulock_wake: ULockWakeFn,
17}
18
19#[cfg(all(
20    target_os = "macos",
21    target_arch = "aarch64",
22    not(feature = "weak-aarch64-macos")
23))]
24mod imp {
25    use super::*;
26    pub(super) static API: ULockApi = ULockApi {
27        ulock_wait: crate::__ulock_wait,
28        ulock_wake: crate::__ulock_wake,
29    };
30
31    #[inline]
32    pub(super) unsafe fn ulock_wait(
33        op: u32,
34        addr: *mut c_void,
35        val: u64,
36        micros: u32,
37    ) -> Result<c_int, ApiUnsupported> {
38        Ok(crate::__ulock_wait(op, addr, val, micros))
39    }
40
41    #[inline]
42    pub(super) unsafe fn ulock_wake(
43        op: u32,
44        addr: *mut c_void,
45        val: u64,
46    ) -> Result<c_int, ApiUnsupported> {
47        Ok(crate::__ulock_wake(op, addr, val))
48    }
49
50    #[inline]
51    pub(super) fn get() -> Result<&'static ULockApi, ApiUnsupported> {
52        Ok(&API)
53    }
54}
55
56#[cfg(not(all(
57    target_os = "macos",
58    target_arch = "aarch64",
59    not(feature = "weak-aarch64-macos")
60)))]
61mod imp {
62    use super::*;
63    use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering::*};
64
65    #[repr(C)]
66    struct MaybeULockApi {
67        ulock_wait: AtomicUsize,
68        ulock_wake: AtomicUsize,
69    }
70
71    const NOT_INIT: u8 = 0;
72    const BAD_INIT: u8 = 1;
73    const GOOD_INIT: u8 = 2;
74    static INIT_STATE: AtomicU8 = AtomicU8::new(NOT_INIT);
75
76    static API: MaybeULockApi = MaybeULockApi {
77        ulock_wait: AtomicUsize::new(0),
78        ulock_wake: AtomicUsize::new(0),
79    };
80
81    #[inline]
82    pub(super) unsafe fn ulock_wait(
83        op: u32,
84        addr: *mut c_void,
85        val: u64,
86        micros: u32,
87    ) -> Result<cty::c_int, ApiUnsupported> {
88        let func: Option<ULockWaitFn> = core::mem::transmute(API.ulock_wait.load(Relaxed));
89        if let Some(func) = func {
90            return Ok((func)(op, addr, val, micros));
91        }
92        ulock_wait_outline(op, addr, val, micros)
93    }
94
95    #[cold]
96    unsafe fn ulock_wait_outline(
97        op: u32,
98        addr: *mut c_void,
99        val: u64,
100        micros: u32,
101    ) -> Result<cty::c_int, ApiUnsupported> {
102        maybe_init().and_then(|_| {
103            let func: ULockWaitFn = core::mem::transmute(API.ulock_wait.load(Relaxed));
104            Ok(func(op, addr, val, micros))
105        })
106    }
107
108    #[inline]
109    pub(super) unsafe fn ulock_wake(
110        op: u32,
111        addr: *mut c_void,
112        val: u64,
113    ) -> Result<cty::c_int, ApiUnsupported> {
114        let func: Option<ULockWakeFn> = core::mem::transmute(API.ulock_wake.load(Relaxed));
115        if let Some(func) = func {
116            return Ok((func)(op, addr, val));
117        }
118        ulock_wake_outline(op, addr, val)
119    }
120
121    #[cold]
122    unsafe fn ulock_wake_outline(
123        op: u32,
124        addr: *mut c_void,
125        val: u64,
126    ) -> Result<cty::c_int, ApiUnsupported> {
127        maybe_init().and_then(|_| {
128            let func: ULockWakeFn = core::mem::transmute(API.ulock_wake.load(Relaxed));
129            Ok(func(op, addr, val))
130        })
131    }
132
133    #[inline]
134    pub(super) fn get() -> Result<&'static super::ULockApi, ApiUnsupported> {
135        maybe_init()?;
136        // Note: we know that `API` will never be written to again.
137        Ok(unsafe { &*(&API as *const _ as *const super::ULockApi) })
138    }
139
140    #[inline]
141    fn maybe_init() -> Result<(), ApiUnsupported> {
142        match INIT_STATE.load(Acquire) {
143            BAD_INIT => return Err(ApiUnsupported(())),
144            GOOD_INIT => {}
145            NOT_INIT => init()?,
146            v => {
147                debug_assert!(false, "unknown state (please report this bug): {}", v);
148                unsafe { core::hint::unreachable_unchecked() };
149            }
150        }
151        debug_assert_eq!(INIT_STATE.load(Acquire), GOOD_INIT);
152        debug_assert!(API.ulock_wait.load(Relaxed) != 0);
153        debug_assert!(API.ulock_wake.load(Relaxed) != 0);
154        Ok(())
155    }
156
157    fn init() -> Result<(), ApiUnsupported> {
158        let ulock_wait = unsafe { load_sym("__ulock_wait\0") as usize };
159        let ulock_wake = unsafe { load_sym("__ulock_wake\0") as usize };
160        if ulock_wait == 0 || ulock_wake == 0 {
161            let old = match INIT_STATE.compare_exchange(NOT_INIT, BAD_INIT, AcqRel, Acquire) {
162                Ok(v) => v,
163                Err(v) => v,
164            };
165            return match old {
166                NOT_INIT => Err(ApiUnsupported(())),
167                GOOD_INIT => Ok(()),
168                BAD_INIT => Err(ApiUnsupported(())),
169                v => {
170                    debug_assert!(false, "unknown state (please report this bug): {}", v);
171                    unsafe { core::hint::unreachable_unchecked() };
172                }
173            };
174        }
175
176        let old = API
177            .ulock_wait
178            .compare_exchange(0, ulock_wait, Relaxed, Relaxed);
179        debug_assert!(old.is_ok() || old == Err(ulock_wait));
180
181        let old = API
182            .ulock_wake
183            .compare_exchange(0, ulock_wake, Relaxed, Relaxed);
184        debug_assert!(old.is_ok() || old == Err(ulock_wake));
185
186        if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
187            // Force bus lock w/ `lock xchg` as an optimization. Note that we're
188            // still sound if this doesn't work, but it can help ensure that
189            // this value is seen by other threads sooner.
190            let old = INIT_STATE.swap(GOOD_INIT, SeqCst);
191            debug_assert_ne!(old, BAD_INIT);
192        } else {
193            INIT_STATE.store(GOOD_INIT, Release);
194        }
195        Ok(())
196    }
197
198    #[inline]
199    unsafe fn load_sym(s: &str) -> *mut c_void {
200        const RTLD_DEFAULT: *mut c_void = -2isize as *mut c_void;
201        extern "C" {
202            fn dlsym(h: *mut c_void, s: *const cty::c_char) -> *mut c_void;
203        }
204        debug_assert_eq!(s.as_bytes()[s.len() - 1], b'\0');
205        dlsym(RTLD_DEFAULT, s.as_ptr().cast())
206    }
207
208    // Try and run the initalization before main. Not required for
209    // correctness/soundness, just helps us always use the fast path.
210    #[used]
211    #[cfg(link_section = "__DATA,__mod_init_func")]
212    static CTOR: extern "C" fn() = init_function;
213    #[allow(dead_code)]
214    extern "C" fn init_function() {
215        drop(init());
216    }
217}
218
219/// Equivalent to [`__ulock_wait`], but lazy-loads and returns
220/// `Err(ApiUnsupported)` if this machine doesn't support the API.
221///
222/// It's generally faster to call this than to call
223/// `ULockApi::get()?.ulock_wait(...)` unless you can cache the result of `get`
224/// somewhere easy.
225///
226/// # Safety
227/// Same as [`__ulock_wait`].
228#[inline]
229pub unsafe fn ulock_wait(
230    op: u32,
231    addr: *mut c_void,
232    val: u64,
233    micros: u32,
234) -> Result<cty::c_int, ApiUnsupported> {
235    imp::ulock_wait(op, addr, val, micros)
236}
237
238/// Equivalent to [`__ulock_wake`], but lazy-loads and returns
239/// `Err(ApiUnsupported)` if this machine doesn't support the API.
240///
241/// It's generally faster to call this than to call
242/// `ULockApi::get()?.ulock_wake(...)` unless you can cache the result of `get`
243/// somewhere easy.
244///
245/// # Safety
246/// Same as [`__ulock_wake`]
247#[inline]
248pub unsafe fn ulock_wake(
249    op: u32,
250    addr: *mut c_void,
251    val: u64,
252) -> Result<cty::c_int, ApiUnsupported> {
253    imp::ulock_wake(op, addr, val)
254}
255
256impl ULockApi {
257    /// Get a reference to the API, if supported, loading it if not already
258    /// loaded. This function is relatively fast for non-first time loads (which
259    /// should be rare), but it's still likely (very slightly) better to call
260    /// [`ulock_wait`] or [`ulock_wake`] unless you cache it somewhere quite
261    /// cheap to access.
262    #[inline]
263    pub fn get() -> Result<&'static Self, ApiUnsupported> {
264        imp::get()
265    }
266}
267
268/// Error indicating that we failed to load the API.
269#[derive(Clone, PartialEq, Copy)]
270pub struct ApiUnsupported(());
271
272impl core::fmt::Display for ApiUnsupported {
273    #[cold]
274    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
275        f.write_str("ulock api unsupported")
276    }
277}
278
279impl core::fmt::Debug for ApiUnsupported {
280    #[cold]
281    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
282        f.write_str("ulock api unsupported")
283    }
284}
ulock_sys/weak.rs

ulock_sys/
weak.rs
