vpp_plugin/vlib/main/sync.rs
1//! Core synchronization primitives for vlib main.
2
3use std::{
4 cell::{Cell, UnsafeCell},
5 fmt,
6 marker::PhantomData,
7 ops::{Deref, DerefMut},
8 ptr::NonNull,
9};
10
11use crate::vlib::{BarrierHeldMainRef, MainRef, main::BarrierGuard};
12
13/// A read/write lock using VPP's barrier to provide exclusion between threads
14///
15/// VPP implements a barrier in the main thread which blocks all worker threads from running. The
16/// `BarrierRwLock` is an abstraction around this which allows a writer in the VPP main thread
17/// whilst the barrier is held and readers in either VPP workers or the VPP main thread.
18///
19/// Taking read or write "locks" are guaranteed to never block - blocking instead occurs in the
20/// VPP main and worker threads when the VPP barrier is taken.
21pub struct BarrierRwLock<T: ?Sized> {
22 /// The number of readers
23 ///
24 /// Note that this doesn't use `AtomicU32` because it's only modified on the VPP main thread.
25 readers: UnsafeCell<u32>,
26 /// Whether there is a writer
27 ///
28 /// Note that this doesn't use `AtomicBool` because it's only modified on the VPP main thread.
29 writer: UnsafeCell<bool>,
30 /// The lock-protected data.
31 data: UnsafeCell<T>,
32}
33
34impl<T> BarrierRwLock<T> {
35 /// Create a new barrier-backed read/write lock.
36 #[inline]
37 pub const fn new(t: T) -> Self {
38 Self {
39 data: UnsafeCell::new(t),
40 readers: UnsafeCell::new(0),
41 writer: UnsafeCell::new(false),
42 }
43 }
44}
45
46impl<T: ?Sized> BarrierRwLock<T> {
47 /// Locks this `BarrierRwLock` with shared read access.
48 ///
49 /// Returns an RAII guard which will release this thread's shared access
50 /// once it is dropped.
51 ///
52 /// # Panics
53 ///
54 /// Panics if a write lock has already been taken by this thread and not dropped.
55 #[inline(always)]
56 pub fn read<'a>(&'a self, vm: &'a MainRef) -> BarrierRwLockReadGuard<'a, T> {
57 let main_thread = vm.thread_index() == 0;
58 // SAFETY: calling `BarrierRwLockReadGuard::new` is valid when we have a reference to the lock
59 // and we are on a known VPP thread. These conditions are satisfied by the public API.
60 unsafe { BarrierRwLockReadGuard::new(self, main_thread) }
61 }
62
63 /// Locks this `BarrierRwLock` with write access.
64 ///
65 /// This is used on the VPP main thread in contexts where the VPP barrier is held.
66 ///
67 /// Returns an RAII guard which will release this thread's access
68 /// once it is dropped.
69 ///
70 /// # Panics
71 ///
72 /// Panics if a read or another write lock has already been taken by this thread and not
73 /// dropped.
74 #[inline(always)]
75 pub fn write<'a>(&'a self, vm: &'a BarrierHeldMainRef) -> BarrierRwLockWriteGuard<'a, T> {
76 // Make sure we match the check in read()
77 debug_assert_eq!(vm.thread_index(), 0);
78 // SAFETY: `BarrierRwLockWriteGuard::new` is only called on the main thread while the
79 // barrier is held.
80 unsafe { BarrierRwLockWriteGuard::new(self) }
81 }
82
83 /// Takes the VPP barrier and locks this `BarrierRwLock` with write access.
84 ///
85 /// This is used on the VPP main thread in contexts where the VPP barrier isn't already held.
86 /// If it is already known to be held, prefer [`BarrierRwLock::write`].
87 ///
88 /// Returns an RAII guard which will release this thread's access, and then the barrier,
89 /// once it is dropped.
90 ///
91 /// The returned guard must not be held across an async await.
92 ///
93 /// # Panics
94 ///
95 /// Panics if not called on the VPP main thread, or if a read or another write lock has
96 /// already been taken by this thread and not dropped.
97 pub fn write_sync<'a>(&'a self, vm: &'a MainRef) -> BarrierRwLockSyncWriteGuard<'a, T> {
98 let barrier = vm.barrier_sync();
99 // SAFETY: `barrier` is held for the lifetime of the returned guard, and `inner` is
100 // dropped before `barrier`.
101 let inner = unsafe { BarrierRwLockWriteGuard::new(self) };
102 BarrierRwLockSyncWriteGuard {
103 inner,
104 _barrier: barrier,
105 }
106 }
107
108 /// Get a mutable reference to the contained data without locking.
109 ///
110 /// This call borrows the `BarrierRwLock` mutably (at compile-time) which guarantees that we
111 /// possess the only reference.
112 pub fn get_mut(&mut self) -> &mut T {
113 self.data.get_mut()
114 }
115
116 /// Returns a raw pointer to the underlying data.
117 ///
118 /// The returned pointer is always non-null and properly aligned, but it is
119 /// the user's responsibility to ensure that any reads and writes through it
120 /// are properly synchronized to avoid data races, and that it is not read
121 /// or written through after the lock is dropped.
122 pub const fn data_ptr(&self) -> *mut T {
123 self.data.get()
124 }
125}
126
127impl<T> BarrierRwLock<T> {
128 /// Consume the lock and return the underlying data.
129 pub fn into_inner(self) -> T {
130 self.data.into_inner()
131 }
132}
133
134// SAFETY: `BarrierRwLock<T>` is safe to send to another thread if `T: Send`.
135unsafe impl<T: ?Sized + Send> Send for BarrierRwLock<T> {}
136
137// SAFETY: `BarrierRwLock<T>` is safe to share between threads if `T: Send + Sync`.
138unsafe impl<T: ?Sized + Send + Sync> Sync for BarrierRwLock<T> {}
139
140impl<T: Default> Default for BarrierRwLock<T> {
141 /// Creates a new `BarrierRwLock<T>`, with the `Default` value for T.
142 fn default() -> BarrierRwLock<T> {
143 BarrierRwLock::new(Default::default())
144 }
145}
146
147/// Shared read guard returned by [`BarrierRwLock::read`].
148pub struct BarrierRwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> {
149 /// A pointer to the data protected by the `BarrierRwLock`. Note that we use a pointer here
150 /// instead of `&'rwlock T` to avoid `noalias` violations, because a `BarrierRwLockReadGuard`
151 /// instance only holds immutability until it drops, not for its whole scope.
152 data: NonNull<T>,
153
154 /// A reference to the [`BarrierRwLock`] that we have read-locked.
155 lock: &'rwlock BarrierRwLock<T>,
156
157 /// Whether the lock is on the VPP main thread or not
158 main_thread: bool,
159}
160
161// Note: Send not implemented here as that would prevent the optimisation of not incrementing
162// readers for VPP worker threads, since the guard could then be sent to the VPP main thread
163// and used to access data while there is a write lock taken, which violates `noalias` rules.
164
165// SAFETY: `BarrierRwLockReadGuard` is immutable references to valid data; `Sync` is safe for T: Sync.
166unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockReadGuard<'_, T> {}
167
168impl<'rwlock, T: ?Sized> BarrierRwLockReadGuard<'rwlock, T> {
169 /// Creates a new instance of `BarrierRwLockReadGuard<T>` from a `BarrierRwLock<T>`.
170 ///
171 /// # Panics
172 ///
173 /// Panics if a write lock has already been taken by this thread and not dropped.
174 ///
175 /// # Safety
176 ///
177 /// This function is safe if and only if called from a thread that VPP barriers know about,
178 /// i.e. either the VPP main thread or a VPP worker thread.
179 #[inline(always)]
180 unsafe fn new(
181 lock: &'rwlock BarrierRwLock<T>,
182 main_thread: bool,
183 ) -> BarrierRwLockReadGuard<'rwlock, T> {
184 // SAFETY: `lock.writer` is valid because `lock` is a valid pointer to a live lock.
185 if main_thread && unsafe { *lock.writer.get() } {
186 panic!("Write lock already taken by this thread");
187 }
188
189 // SAFETY: `lock.data` is valid and aligned, and lock lifetime guarantees it outlives the guard.
190 let data = unsafe { NonNull::new_unchecked(lock.data.get()) };
191 if main_thread {
192 // SAFETY: Only main thread increments/decrements readers so there is no data race.
193 unsafe {
194 *lock.readers.get() += 1;
195 }
196 }
197 Self {
198 data,
199 lock,
200 main_thread,
201 }
202 }
203}
204
205impl<T: ?Sized> Drop for BarrierRwLockReadGuard<'_, T> {
206 #[inline(always)]
207 fn drop(&mut self) {
208 if self.main_thread {
209 // SAFETY: Only main thread mutates `readers` so there is no data race. We are on
210 // the main thread by conditional.
211 unsafe {
212 *self.lock.readers.get() -= 1;
213 }
214 }
215 }
216}
217
218impl<T: ?Sized> Deref for BarrierRwLockReadGuard<'_, T> {
219 type Target = T;
220
221 #[inline(always)]
222 fn deref(&self) -> &T {
223 // SAFETY: the conditions of `BarrierRwLockReadGuard::new` were satisfied when created.
224 unsafe { self.data.as_ref() }
225 }
226}
227
228impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockReadGuard<'_, T> {
229 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
230 (**self).fmt(f)
231 }
232}
233
234impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockReadGuard<'_, T> {
235 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
236 (**self).fmt(f)
237 }
238}
239
240/// Exclusive write guard returned by [`BarrierRwLock::write`].
241pub struct BarrierRwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> {
242 /// A reference to the [`RwLock`] that we have write-locked.
243 lock: &'rwlock BarrierRwLock<T>,
244
245 /// Prevent the type from being Send
246 _phantom: PhantomData<Cell<()>>,
247}
248
249impl<'rwlock, T: ?Sized> BarrierRwLockWriteGuard<'rwlock, T> {
250 /// Creates a new instance of `BarrierRwLockWriteGuard<T>` from a `BarrierRwLock<T>`.
251 ///
252 /// # Panics
253 ///
254 /// Panics if a read or another write lock has already been taken by this thread and not
255 /// dropped.
256 ///
257 /// # Safety
258 ///
259 /// This function is safe if and only if the same thread is holding the VPP barrier prior to
260 /// calling this function and continues to hold it for the lifetime of this object.
261 #[inline(always)]
262 unsafe fn new(lock: &'rwlock BarrierRwLock<T>) -> BarrierRwLockWriteGuard<'rwlock, T> {
263 // SAFETY: this function is only called with barrier held and no concurrent write.
264 unsafe {
265 if *lock.readers.get() != 0 {
266 panic!("Read lock already taken by this thread");
267 }
268 if *lock.writer.get() {
269 panic!("Write lock already taken by this thread");
270 }
271 *lock.writer.get() = true;
272 }
273 BarrierRwLockWriteGuard {
274 lock,
275 _phantom: PhantomData,
276 }
277 }
278}
279
280impl<T: ?Sized> Drop for BarrierRwLockWriteGuard<'_, T> {
281 #[inline(always)]
282 fn drop(&mut self) {
283 // SAFETY: This is the only writer and barrier is held while the guard is alive.
284 unsafe {
285 *self.lock.writer.get() = false;
286 }
287 }
288}
289
290// Note: no Send implementation as it's not safe to modify `self.lock.writer` on Drop and sending
291// the write guard across threads has limited usefulness.
292
293// SAFETY: `BarrierRwLockWriteGuard` ensures exclusive write access to the protected data
294// during its lifetime via VPP's barrier mechanism, which prevents concurrent access from
295// worker threads. For `T: Sync`, the guard can be safely shared across threads because
296// the underlying data is `Sync` and the barrier guarantees no conflicting accesses occur.
297unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockWriteGuard<'_, T> {}
298
299impl<T: ?Sized> Deref for BarrierRwLockWriteGuard<'_, T> {
300 type Target = T;
301
302 #[inline(always)]
303 fn deref(&self) -> &T {
304 // SAFETY: the conditions of `BarrierRwLockWriteGuard::new` were satisfied when created.
305 unsafe { &*self.lock.data.get() }
306 }
307}
308
309impl<T: ?Sized> DerefMut for BarrierRwLockWriteGuard<'_, T> {
310 #[inline(always)]
311 fn deref_mut(&mut self) -> &mut T {
312 // SAFETY: the conditions of `BarrierRwLockWriteGuard::new` were satisfied when created.
313 unsafe { &mut *self.lock.data.get() }
314 }
315}
316
317impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockWriteGuard<'_, T> {
318 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
319 (**self).fmt(f)
320 }
321}
322
323impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockWriteGuard<'_, T> {
324 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
325 (**self).fmt(f)
326 }
327}
328
329/// Exclusive write guard that also holds the VPP barrier, returned by
330/// [`BarrierRwLock::write_sync`].
331pub struct BarrierRwLockSyncWriteGuard<'a, T: ?Sized + 'a> {
332 // Note: field order matters - `inner` must drop (clearing `writer`) before the barrier is
333 // released.
334 inner: BarrierRwLockWriteGuard<'a, T>,
335 _barrier: BarrierGuard<'a>,
336}
337
338// SAFETY: as for `BarrierRwLockWriteGuard`, a shared reference only gives out `&T`, and the
339// barrier keeps worker threads stopped for as long as that reference can live. `BarrierGuard`
340// isn't `Sync` because it hands out a `&BarrierHeldMainRef`, which must not reach a thread other
341// than the one owning the `vlib_main_t`. `_barrier` is private and never exposed, so this
342// remains sound only while no accessor for it is added.
343unsafe impl<T: ?Sized + Sync> Sync for BarrierRwLockSyncWriteGuard<'_, T> {}
344
345impl<T: ?Sized> Deref for BarrierRwLockSyncWriteGuard<'_, T> {
346 type Target = T;
347
348 #[inline(always)]
349 fn deref(&self) -> &T {
350 &self.inner
351 }
352}
353
354impl<T: ?Sized> DerefMut for BarrierRwLockSyncWriteGuard<'_, T> {
355 #[inline(always)]
356 fn deref_mut(&mut self) -> &mut T {
357 &mut self.inner
358 }
359}
360
361impl<T: ?Sized + fmt::Debug> fmt::Debug for BarrierRwLockSyncWriteGuard<'_, T> {
362 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
363 (**self).fmt(f)
364 }
365}
366
367impl<T: ?Sized + fmt::Display> fmt::Display for BarrierRwLockSyncWriteGuard<'_, T> {
368 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
369 (**self).fmt(f)
370 }
371}
372
373#[cfg(test)]
374mod tests {
375 use std::thread;
376
377 use crate::{
378 bindings::vlib_main_t,
379 vlib::{BarrierHeldMainRef, MainRef, main::sync::BarrierRwLock},
380 };
381
382 #[test]
383 fn concurrent_reads() {
384 let lock = BarrierRwLock::new("value".to_string());
385 let ref_lock = &lock;
386 thread::scope(|s| {
387 let thread1 = s.spawn(move || {
388 let mut main = vlib_main_t::default();
389 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
390 // call.
391 let main_ref = unsafe { MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
392 for _ in 0..1000 {
393 assert_eq!(*ref_lock.read(main_ref), "value");
394 }
395 });
396 let thread2 = s.spawn(move || {
397 let mut main = vlib_main_t {
398 thread_index: 1,
399 ..vlib_main_t::default()
400 };
401 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
402 // call.
403 let main_ref = unsafe { MainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
404 for _ in 0..1000 {
405 assert_eq!(*ref_lock.read(main_ref), "value");
406 }
407 });
408 thread1.join().unwrap();
409 thread2.join().unwrap();
410 });
411 }
412
413 #[test]
414 fn write_guard() {
415 let mut main = vlib_main_t::default();
416 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
417 // call.
418 let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
419 let lock = BarrierRwLock::new("value".to_string());
420 *lock.write(main_ref) = "new value".to_string();
421 assert_eq!(*lock.read(main_ref), "new value");
422 }
423
424 #[test]
425 #[should_panic(expected = "Write lock already taken by this thread")]
426 fn read_and_write1() {
427 let mut main = vlib_main_t::default();
428 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
429 // call.
430 let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
431 let lock = BarrierRwLock::new("value".to_string());
432 let _guard1 = lock.write(main_ref);
433 let _guard2 = lock.read(main_ref);
434 }
435
436 #[test]
437 #[should_panic(expected = "Read lock already taken by this thread")]
438 fn read_and_write2() {
439 let mut main = vlib_main_t::default();
440 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
441 // call.
442 let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
443 let lock = BarrierRwLock::new("value".to_string());
444 let _guard1 = lock.read(main_ref);
445 let _guard2 = lock.write(main_ref);
446 }
447
448 #[test]
449 #[should_panic(expected = "Write lock already taken by this thread")]
450 fn write_write() {
451 let mut main = vlib_main_t::default();
452 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
453 // call.
454 let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
455 let lock = BarrierRwLock::new("value".to_string());
456 let _guard1 = lock.write(main_ref);
457 let _guard2 = lock.write(main_ref);
458 }
459
460 /// Test misc small utilities of [`BarrierRwLock`]
461 #[test]
462 fn misc() {
463 let mut main = vlib_main_t::default();
464 // SAFETY: main is sufficiently initialised for the test and valid for the duration of the
465 // call.
466 let main_ref = unsafe { BarrierHeldMainRef::from_ptr_mut(std::ptr::addr_of_mut!(main)) };
467 let mut lock: BarrierRwLock<String> = BarrierRwLock::default();
468
469 assert_eq!(*lock.write(main_ref), "");
470
471 *lock.get_mut() = "value".to_string();
472
473 assert_eq!(lock.write(main_ref).to_string(), "value");
474 assert_eq!(format!("{:?}", lock.write(main_ref)), "\"value\"");
475 assert_eq!(lock.read(main_ref).to_string(), "value");
476 assert_eq!(format!("{:?}", lock.read(main_ref)), "\"value\"");
477
478 // SAFETY: data_ptr() returns a valid pointer and it remains valid throughout its use
479 unsafe {
480 assert_eq!(&*lock.data_ptr(), "value");
481 }
482
483 assert_eq!(lock.into_inner(), "value");
484 }
485}