Skip to main content

vpp_plugin/vlib/main/
sync.rs

1//! Core synchronization primitives for vlib main.
2
3use std::{
4    cell::{Cell, UnsafeCell},
5    fmt,
6    marker::PhantomData,
7    ops::{Deref, DerefMut},
8    ptr::NonNull,
9};
10
11use crate::vlib::{BarrierHeldMainRef, MainRef, main::BarrierGuard};
12
13/// A read/write lock using VPP's barrier to provide exclusion between threads
14///
15/// VPP implements a barrier in the main thread which blocks all worker threads from running. The
16/// `BarrierRwLock` is an abstraction around this which allows a writer in the VPP main thread
17/// whilst the barrier is held and readers in either VPP workers or the VPP main thread.
18///
19/// Taking read or write "locks" are guaranteed to never block - blocking instead occurs in the
20/// VPP main and worker threads when the VPP barrier is taken.
21pub struct BarrierRwLock<T: ?Sized> {
22    /// The number of readers
23    ///
24    /// Note that this doesn't use `AtomicU32` because it's only modified on the VPP main thread.
25    readers: UnsafeCell<u32>,
26    /// Whether there is a writer
27    ///
28    /// Note that this doesn't use `AtomicBool` because it's only modified on the VPP main thread.
29    writer: UnsafeCell<bool>,
30    /// The lock-protected data.
31    data: UnsafeCell<T>,
32}
33
34impl<T> BarrierRwLock<T> {
35    /// Create a new barrier-backed read/write lock.
36    #[inline]
37    pub const fn new(t: T) -> Self {
38        Self {
39            data: UnsafeCell::new(t),
40            readers: UnsafeCell::new(0),
41            writer: UnsafeCell::new(false),
42        }
43    }
44}
45
46impl<T: ?Sized> BarrierRwLock<T> {
47    /// Locks this `BarrierRwLock` with shared read access.
48    ///
49    /// Returns an RAII guard which will release this thread's shared access
50    /// once it is dropped.
51    ///
52    /// # Panics
53    ///
54    /// Panics if a write lock has already been taken by this thread and not dropped.
55    #[inline(always)]
56    pub fn read<'a>(&'a self, vm: &'a MainRef) -> BarrierRwLockReadGuard<'a, T> {
57        let main_thread = vm.thread_index() == 0;
58        // SAFETY: calling `BarrierRwLockReadGuard::new` is valid when we have a reference to the lock
59        // and we are on a known VPP thread. These conditions are satisfied by the public API.
60        unsafe { BarrierRwLockReadGuard::new(self, main_thread) }
61    }
62
63    /// Locks this `BarrierRwLock` with write access.
64    ///
65    /// This is used on the VPP main thread in contexts where the VPP barrier is held.
66    ///
67    /// Returns an RAII guard which will release this thread's access
68    /// once it is dropped.
69    ///
70    /// # Panics
71    ///
72    /// Panics if a read or another write lock has already been taken by this thread and not
73    /// dropped.
74    #[inline(always)]
75    pub fn write<'a>(&'a self, vm: &'a BarrierHeldMainRef) -> BarrierRwLockWriteGuard<'a, T> {
76        // Make sure we match the check in read()
77        debug_assert_eq!(vm.thread_index(), 0);
78        // SAFETY: `BarrierRwLockWriteGuard::new` is only called on the main thread while the
79        // barrier is held.
80        unsafe { BarrierRwLockWriteGuard::new(self) }
81    }
82
83    /// Takes the VPP barrier and locks this `BarrierRwLock` with write access.
84    ///
85    /// This is used on the VPP main thread in contexts where the VPP barrier isn't already held.
86    /// If it is already known to be held, prefer [`BarrierRwLock::write`].
87    ///
88    /// Returns an RAII guard which will release this thread's access, and then the barrier,
89    /// once it is dropped.
90    ///
91    /// The returned guard must not be held across an async await.
92    ///
93    /// # Panics
94    ///
95    /// Panics if not called on the VPP main thread, or if a read or another write lock has
96    /// already been taken by this thread and not dropped.
97    pub fn write_sync<'a>(&'a self, vm: &'a MainRef) -> BarrierRwLockSyncWriteGuard<'a, T> {
98        let barrier = vm.barrier_sync();
99        // SAFETY: `barrier` is held for the lifetime of the returned guard, and `inner` is
100        // dropped before `barrier`.
101        let inner = unsafe { BarrierRwLockWriteGuard::new(self) };
102        BarrierRwLockSyncWriteGuard {
103            inner,
104            _barrier: barrier,
105        }
106    }
107
108    /// Get a mutable reference to the contained data without locking.
109    ///
110    /// This call borrows the `BarrierRwLock` mutably (at compile-time) which guarantees that we
111    /// possess the only reference.
112    pub fn get_mut(&mut self) -> &mut T {
113        self.data.get_mut()
114    }
115
116    /// Returns a raw pointer to the underlying data.
117    ///
118    /// The returned pointer is always non-null and properly aligned, but it is
119    /// the user's responsibility to ensure that any reads and writes through it
120    /// are properly synchronized to avoid data races, and that it is not read
121    /// or written through after the lock is dropped.
122    pub const fn data_ptr(&self) -> *mut T {
123        self.data.get()
124    }
125}
126
127impl<T> BarrierRwLock<T> {
128    /// Consume the lock and return the underlying data.
129    pub fn into_inner(self) -> T {
130        self.data.into_inner()
131    }
132}
133
134// SAFETY: `BarrierRwLock<T>` is safe to send to another thread if `T: Send`.
135unsafe impl<T: ?Sized + Send> Send for BarrierRwLock<T> {}
136
137// SAFETY: `BarrierRwLock<T>` is safe to share between threads if `T: Send + Sync`.
138unsafe impl<T: ?Sized + Send + Sync> Sync for BarrierRwLock<T> {}
139
140impl<T: Default> Default for BarrierRwLock<T> {
141    /// Creates a new `BarrierRwLock<T>`, with the `Default` value for T.
142    fn default() -> BarrierRwLock<T> {
143        BarrierRwLock::new(Default::default())
144    }
145}
146
147/// Shared read guard returned by [`BarrierRwLock::read`].
148pub struct BarrierRwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> {
149    /// A pointer to the data protected by the `BarrierRwLock`. Note that we use a pointer here
150    /// instead of `&'rwlock T` to avoid `noalias` violations, because a `BarrierRwLockReadGuard`
151    /// instance only holds immutability until it drops, not for its whole scope.
152    data: NonNull<T>,
153
154    /// A reference to the [`BarrierRwLock`] that we have read-locked.
155    lock: &'rwlock BarrierRwLock<T>,
156
157    /// Whether the lock is on the VPP main thread or not
158    main_thread: bool,
159}
160
161// Note: Send not implemented here as that would prevent the optimisation of not incrementing
162// readers for VPP worker threads, since the guard could then be sent to the VPP main thread
163// and used to access data while there is a write lock taken, which violates `noalias` rules.
164
165// SAFETY: `BarrierRwLockReadGuard` is immutable references to valid data; `Sync` is safe for T: Sync.
166unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockReadGuard<'_, T> {}
167
168impl<'rwlock, T: ?Sized> BarrierRwLockReadGuard<'rwlock, T> {
169    /// Creates a new instance of `BarrierRwLockReadGuard<T>` from a `BarrierRwLock<T>`.
170    ///
171    /// # Panics
172    ///
173    /// Panics if a write lock has already been taken by this thread and not dropped.
174    ///
175    /// # Safety
176    ///
177    /// This function is safe if and only if called from a thread that VPP barriers know about,
178    /// i.e. either the VPP main thread or a VPP worker thread.
179    #[inline(always)]
180    unsafe fn new(
181        lock: &'rwlock BarrierRwLock<T>,
182        main_thread: bool,
183    ) -> BarrierRwLockReadGuard<'rwlock, T> {
184        // SAFETY: `lock.writer` is valid because `lock` is a valid pointer to a live lock.
185        if main_thread && unsafe { *lock.writer.get() } {
186            panic!("Write lock already taken by this thread");
187        }
188
189        // SAFETY: `lock.data` is valid and aligned, and lock lifetime guarantees it outlives the guard.
190        let data = unsafe { NonNull::new_unchecked(lock.data.get()) };
191        if main_thread {
192            // SAFETY: Only main thread increments/decrements readers so there is no data race.
193            unsafe {
194                *lock.readers.get() += 1;
195            }
196        }
197        Self {
198            data,
199            lock,
200            main_thread,
201        }
202    }
203}
204
205impl<T: ?Sized> Drop for BarrierRwLockReadGuard<'_, T> {
206    #[inline(always)]
207    fn drop(&mut self) {
208        if self.main_thread {
209            // SAFETY: Only main thread mutates `readers` so there is no data race. We are on
210            // the main thread by conditional.
211            unsafe {
212                *self.lock.readers.get() -= 1;
213            }
214        }
215    }
216}
217
218impl<T: ?Sized> Deref for BarrierRwLockReadGuard<'_, T> {
219    type Target = T;
220
221    #[inline(always)]
222    fn deref(&self) -> &T {
223        // SAFETY: the conditions of `BarrierRwLockReadGuard::new` were satisfied when created.
224        unsafe { self.data.as_ref() }
225    }
226}
227
228impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockReadGuard<'_, T> {
229    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
230        (**self).fmt(f)
231    }
232}
233
234impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockReadGuard<'_, T> {
235    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
236        (**self).fmt(f)
237    }
238}
239
240/// Exclusive write guard returned by [`BarrierRwLock::write`].
241pub struct BarrierRwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> {
242    /// A reference to the [`RwLock`] that we have write-locked.
243    lock: &'rwlock BarrierRwLock<T>,
244
245    /// Prevent the type from being Send
246    _phantom: PhantomData<Cell<()>>,
247}
248
249impl<'rwlock, T: ?Sized> BarrierRwLockWriteGuard<'rwlock, T> {
250    /// Creates a new instance of `BarrierRwLockWriteGuard<T>` from a `BarrierRwLock<T>`.
251    ///
252    /// # Panics
253    ///
254    /// Panics if a read or another write lock has already been taken by this thread and not
255    /// dropped.
256    ///
257    /// # Safety
258    ///
259    /// This function is safe if and only if the same thread is holding the VPP barrier prior to
260    /// calling this function and continues to hold it for the lifetime of this object.
261    #[inline(always)]
262    unsafe fn new(lock: &'rwlock BarrierRwLock<T>) -> BarrierRwLockWriteGuard<'rwlock, T> {
263        // SAFETY: this function is only called with barrier held and no concurrent write.
264        unsafe {
265            if *lock.readers.get() != 0 {
266                panic!("Read lock already taken by this thread");
267            }
268            if *lock.writer.get() {
269                panic!("Write lock already taken by this thread");
270            }
271            *lock.writer.get() = true;
272        }
273        BarrierRwLockWriteGuard {
274            lock,
275            _phantom: PhantomData,
276        }
277    }
278}
279
280impl<T: ?Sized> Drop for BarrierRwLockWriteGuard<'_, T> {
281    #[inline(always)]
282    fn drop(&mut self) {
283        // SAFETY: This is the only writer and barrier is held while the guard is alive.
284        unsafe {
285            *self.lock.writer.get() = false;
286        }
287    }
288}
289
290// Note: no Send implementation as it's not safe to modify `self.lock.writer` on Drop and sending
291// the write guard across threads has limited usefulness.
292
293// SAFETY: `BarrierRwLockWriteGuard` ensures exclusive write access to the protected data
294// during its lifetime via VPP's barrier mechanism, which prevents concurrent access from
295// worker threads. For `T: Sync`, the guard can be safely shared across threads because
296// the underlying data is `Sync` and the barrier guarantees no conflicting accesses occur.
297unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockWriteGuard<'_, T> {}
298
299impl<T: ?Sized> Deref for BarrierRwLockWriteGuard<'_, T> {
300    type Target = T;
301
302    #[inline(always)]
303    fn deref(&self) -> &T {
304        // SAFETY: the conditions of `BarrierRwLockWriteGuard::new` were satisfied when created.
305        unsafe { &*self.lock.data.get() }
306    }
307}
308
309impl<T: ?Sized> DerefMut for BarrierRwLockWriteGuard<'_, T> {
310    #[inline(always)]
311    fn deref_mut(&mut self) -> &mut T {
312        // SAFETY: the conditions of `BarrierRwLockWriteGuard::new` were satisfied when created.
313        unsafe { &mut *self.lock.data.get() }
314    }
315}
316
317impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockWriteGuard<'_, T> {
318    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
319        (**self).fmt(f)
320    }
321}
322
323impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockWriteGuard<'_, T> {
324    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
325        (**self).fmt(f)
326    }
327}
328
329/// Exclusive write guard that also holds the VPP barrier, returned by
330/// [`BarrierRwLock::write_sync`].
331pub struct BarrierRwLockSyncWriteGuard<'a, T: ?Sized + 'a> {
332    // Note: field order matters - `inner` must drop (clearing `writer`) before the barrier is
333    // released.
334    inner: BarrierRwLockWriteGuard<'a, T>,
335    _barrier: BarrierGuard<'a>,
336}
337
338// SAFETY: as for `BarrierRwLockWriteGuard`, a shared reference only gives out `&T`, and the
339// barrier keeps worker threads stopped for as long as that reference can live. `BarrierGuard`
340// isn't `Sync` because it hands out a `&BarrierHeldMainRef`, which must not reach a thread other
341// than the one owning the `vlib_main_t`. `_barrier` is private and never exposed, so this
342// remains sound only while no accessor for it is added.
343unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockSyncWriteGuard<'_, T> {}
344
345impl<T: ?Sized> Deref for BarrierRwLockSyncWriteGuard<'_, T> {
346    type Target = T;
347
348    #[inline(always)]
349    fn deref(&self) -> &T {
350        &self.inner
351    }
352}
353
354impl<T: ?Sized> DerefMut for BarrierRwLockSyncWriteGuard<'_, T> {
355    #[inline(always)]
356    fn deref_mut(&mut self) -> &mut T {
357        &mut self.inner
358    }
359}
360
361impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockSyncWriteGuard<'_, T> {
362    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
363        (**self).fmt(f)
364    }
365}
366
367impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockSyncWriteGuard<'_, T> {
368    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
369        (**self).fmt(f)
370    }
371}
372
373#[cfg(test)]
374mod tests {
375    use std::thread;
376
377    use crate::{
378        bindings::vlib_main_t,
379        vlib::{BarrierHeldMainRef, MainRef, main::sync::BarrierRwLock},
380    };
381
382    #[test]
383    fn concurrent_reads() {
384        let lock = BarrierRwLock::new("value".to_string());
385        let ref_lock = &lock;
386        thread::scope(|s| {
387            let thread1 = s.spawn(move || {
388                let mut main = vlib_main_t::default();
389                // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
390                // call.
391                let main_ref = unsafe { MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
392                for _ in 0..1000 {
393                    assert_eq!(*ref_lock.read(main_ref), "value");
394                }
395            });
396            let thread2 = s.spawn(move || {
397                let mut main = vlib_main_t {
398                    thread_index: 1,
399                    ..vlib_main_t::default()
400                };
401                // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
402                // call.
403                let main_ref = unsafe { MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
404                for _ in 0..1000 {
405                    assert_eq!(*ref_lock.read(main_ref), "value");
406                }
407            });
408            thread1.join().unwrap();
409            thread2.join().unwrap();
410        });
411    }
412
413    #[test]
414    fn write_guard() {
415        let mut main = vlib_main_t::default();
416        // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
417        // call.
418        let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
419        let lock = BarrierRwLock::new("value".to_string());
420        *lock.write(main_ref) = "new value".to_string();
421        assert_eq!(*lock.read(main_ref), "new value");
422    }
423
424    #[test]
425    #[should_panic(expected = "Write lock already taken by this thread")]
426    fn read_and_write1() {
427        let mut main = vlib_main_t::default();
428        // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
429        // call.
430        let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
431        let lock = BarrierRwLock::new("value".to_string());
432        let _guard1 = lock.write(main_ref);
433        let _guard2 = lock.read(main_ref);
434    }
435
436    #[test]
437    #[should_panic(expected = "Read lock already taken by this thread")]
438    fn read_and_write2() {
439        let mut main = vlib_main_t::default();
440        // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
441        // call.
442        let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
443        let lock = BarrierRwLock::new("value".to_string());
444        let _guard1 = lock.read(main_ref);
445        let _guard2 = lock.write(main_ref);
446    }
447
448    #[test]
449    #[should_panic(expected = "Write lock already taken by this thread")]
450    fn write_write() {
451        let mut main = vlib_main_t::default();
452        // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
453        // call.
454        let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
455        let lock = BarrierRwLock::new("value".to_string());
456        let _guard1 = lock.write(main_ref);
457        let _guard2 = lock.write(main_ref);
458    }
459
460    /// Test misc small utilities of [`BarrierRwLock`]
461    #[test]
462    fn misc() {
463        let mut main = vlib_main_t::default();
464        // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
465        // call.
466        let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
467        let mut lock: BarrierRwLock<String> = BarrierRwLock::default();
468
469        assert_eq!(*lock.write(main_ref), "");
470
471        *lock.get_mut() = "value".to_string();
472
473        assert_eq!(lock.write(main_ref).to_string(), "value");
474        assert_eq!(format!("{:?}", lock.write(main_ref)), "\"value\"");
475        assert_eq!(lock.read(main_ref).to_string(), "value");
476        assert_eq!(format!("{:?}", lock.read(main_ref)), "\"value\"");
477
478        // SAFETY: data_ptr() returns a valid pointer and it remains valid throughout its use
479        unsafe {
480            assert_eq!(&*lock.data_ptr(), "value");
481        }
482
483        assert_eq!(lock.into_inner(), "value");
484    }
485}