atomig/
impls.rs

1//! Traits for abstracting over `std` atomics. Mostly hidden implementation detail.
2//!
3//! Most items of these traits are hidden and not part of the public API of this library.
4//! You cannot implement these traits yourself.
5
6use core::{num::Wrapping, sync::atomic::{self, Ordering}};
7use super::{Atom, AtomLogic, AtomInteger};
8
9
10// ===============================================================================================
11// ===== All `Atomic*Impl` traits and `PrimitiveAtom`
12// ===============================================================================================
13
14mod sealed {
15    /// You cannot implement this trait. That is the point.
16    pub trait Sealed {}
17}
18
19/// Primitive types that can directly be used in an atomic way.
20///
21/// This trait is implemented exactly for every type that has a corresponding
22/// atomic type in `std::sync::atomic`. You cannot implement this trait for
23/// your own types; see [`Atom`] instead. This trait's items are not part of
24/// the public API -- see the module docs.
25pub trait PrimitiveAtom: Sized + Copy + sealed::Sealed {
26    /// The standard library type that is the atomic version of `Self`.
27    type Impl: Send + Sync + Unpin;
28
29    #[doc(hidden)]
30    fn into_impl(self) -> Self::Impl;
31    #[doc(hidden)]
32    fn from_impl(imp: Self::Impl) -> Self;
33
34    #[doc(hidden)]
35    fn get_mut(imp: &mut Self::Impl) -> &mut Self;
36    #[doc(hidden)]
37    fn load(imp: &Self::Impl, order: Ordering) -> Self;
38    #[doc(hidden)]
39    fn store(imp: &Self::Impl, v: Self, order: Ordering);
40
41    #[doc(hidden)]
42    fn swap(imp: &Self::Impl, v: Self, order: Ordering) -> Self;
43
44    #[doc(hidden)]
45    fn compare_exchange(
46        imp: &Self::Impl,
47        current: Self,
48        new: Self,
49        success: Ordering,
50        failure: Ordering,
51    ) -> Result<Self, Self>;
52
53    #[doc(hidden)]
54    fn compare_exchange_weak(
55        imp: &Self::Impl,
56        current: Self,
57        new: Self,
58        success: Ordering,
59        failure: Ordering,
60    ) -> Result<Self, Self>;
61
62    #[doc(hidden)]
63    fn fetch_update<F>(
64        imp: &Self::Impl,
65        set_order: Ordering,
66        fetch_order: Ordering,
67        f: F,
68    ) -> Result<Self, Self>
69    where
70        F: FnMut(Self) -> Option<Self>;
71}
72
73/// Atomic types from `std::sync::atomic` which support logical operations.
74///
75/// You cannot implement this trait for your own types; see [`AtomLogic`]
76/// instead. This trait's items are not part of the public API -- see the
77/// module docs.
78pub trait PrimitiveAtomLogic: PrimitiveAtom {
79    #[doc(hidden)]
80    fn fetch_and(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
81    #[doc(hidden)]
82    fn fetch_nand(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
83    #[doc(hidden)]
84    fn fetch_or(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
85    #[doc(hidden)]
86    fn fetch_xor(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
87}
88
89/// Atomic types from `std::sync::atomic` which support integer operations.
90///
91/// You cannot implement this trait for your own types; see [`AtomInteger`]
92/// instead. This trait's items are not part of the public API -- see the
93/// module docs.
94pub trait PrimitiveAtomInteger: PrimitiveAtom {
95    #[doc(hidden)]
96    fn fetch_add(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
97    #[doc(hidden)]
98    fn fetch_sub(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
99
100    #[doc(hidden)]
101    fn fetch_max(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
102    #[doc(hidden)]
103    fn fetch_min(imp: &Self::Impl, val: Self, order: Ordering) -> Self;
104}
105
106
107
108// ===============================================================================================
109// ===== Implementations for standard library types
110// ===============================================================================================
111
112/// Expands to the `pack` and `unpack` methods implemented as ID function.
113macro_rules! id_pack_unpack {
114    () => {
115        fn pack(self) -> Self::Repr {
116            self
117        }
118        fn unpack(src: Self::Repr) -> Self {
119            src
120        }
121    };
122}
123
124/// Expands to all methods from `AtomicImpl`, each forwarding to
125/// `self.that_method`.
126macro_rules! pass_through_methods {
127    ($ty:ty) => {
128        #[inline(always)]
129        fn into_impl(self) -> Self::Impl {
130            <$ty>::new(self)
131        }
132
133        #[inline(always)]
134        fn from_impl(imp: Self::Impl) -> Self {
135            imp.into_inner()
136        }
137
138        #[inline(always)]
139        fn get_mut(imp: &mut Self::Impl) -> &mut Self {
140            imp.get_mut()
141        }
142
143        #[inline(always)]
144        fn load(imp: &Self::Impl, order: Ordering) -> Self {
145            imp.load(order)
146        }
147
148        #[inline(always)]
149        fn store(imp: &Self::Impl, v: Self, order: Ordering) {
150            imp.store(v, order)
151        }
152
153        #[inline(always)]
154        fn swap(imp: &Self::Impl, v: Self, order: Ordering) -> Self {
155            imp.swap(v, order)
156        }
157
158        #[inline(always)]
159        fn compare_exchange(
160            imp: &Self::Impl,
161            current: Self,
162            new: Self,
163            success: Ordering,
164            failure: Ordering,
165        ) -> Result<Self, Self> {
166            imp.compare_exchange(current, new, success, failure)
167        }
168
169        #[inline(always)]
170        fn compare_exchange_weak(
171            imp: &Self::Impl,
172            current: Self,
173            new: Self,
174            success: Ordering,
175            failure: Ordering,
176        ) -> Result<Self, Self> {
177            imp.compare_exchange_weak(current, new, success, failure)
178        }
179
180        fn fetch_update<F>(
181            imp: &Self::Impl,
182            set_order: Ordering,
183            fetch_order: Ordering,
184            f: F,
185        ) -> Result<Self, Self>
186        where
187            F: FnMut(Self) -> Option<Self>
188        {
189            imp.fetch_update(set_order, fetch_order, f)
190        }
191    };
192}
193
194/// Expands to all methods from `AtomicLogicImpl`, each forwarding to
195/// `self.that_method`.
196macro_rules! logical_pass_through_methods {
197    () => {
198        #[inline(always)]
199        fn fetch_and(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
200            imp.fetch_and(val, order)
201        }
202
203        #[inline(always)]
204        fn fetch_nand(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
205            imp.fetch_nand(val, order)
206        }
207
208        #[inline(always)]
209        fn fetch_or(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
210            imp.fetch_or(val, order)
211        }
212
213        #[inline(always)]
214        fn fetch_xor(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
215            imp.fetch_xor(val, order)
216        }
217    };
218}
219
220/// Expands to all methods from `AtomicIntegerImpl`, each forwarding to
221/// `self.that_method`.
222macro_rules! integer_pass_through_methods {
223    () => {
224        #[inline(always)]
225        fn fetch_add(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
226            imp.fetch_add(val, order)
227        }
228
229        #[inline(always)]
230        fn fetch_sub(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
231            imp.fetch_sub(val, order)
232        }
233
234        fn fetch_max(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
235            imp.fetch_max(val, order)
236        }
237
238        fn fetch_min(imp: &Self::Impl, val: Self, order: Ordering) -> Self {
239            imp.fetch_min(val, order)
240        }
241    };
242}
243
244// ----- `*mut T` and `AtomicPtr` -----
245#[cfg(target_has_atomic = "ptr")]
246impl<T> Atom for *mut T {
247    type Repr = Self;
248    id_pack_unpack!();
249}
250
251#[cfg(target_has_atomic = "ptr")]
252impl<T> sealed::Sealed for *mut T {}
253
254#[cfg(target_has_atomic = "ptr")]
255impl<T> PrimitiveAtom for *mut T {
256    type Impl = atomic::AtomicPtr<T>;
257    pass_through_methods!(atomic::AtomicPtr<T>);
258}
259
260
261
262// ----- Integers and `bool` -----
263
264macro_rules! impl_std_atomics {
265    ($ty:ty, $non_zero_ty:ident, $impl_ty:ident, $is_int:ident) => {
266        impl Atom for $ty {
267            type Repr = Self;
268            id_pack_unpack!();
269        }
270
271        impl sealed::Sealed for $ty {}
272        impl AtomLogic for $ty {}
273        impl PrimitiveAtom for $ty {
274            type Impl = atomic::$impl_ty;
275            pass_through_methods!(atomic::$impl_ty);
276        }
277
278        impl PrimitiveAtomLogic for $ty {
279            logical_pass_through_methods!();
280        }
281
282        impl_std_atomics!(@int_methods $ty, $non_zero_ty, $impl_ty, $is_int);
283    };
284    (@int_methods $ty:ty, $non_zero_ty:ident, $impl_ty:ident, true) => {
285        impl AtomInteger for $ty {}
286        impl PrimitiveAtomInteger for $ty {
287            integer_pass_through_methods!();
288        }
289
290        impl Atom for core::num::$non_zero_ty {
291            type Repr = $ty;
292            fn pack(self) -> Self::Repr {
293                self.get()
294            }
295
296            fn unpack(src: Self::Repr) -> Self {
297                // Since `AtomLogic` and `AtomInteger` is not implemented for
298                // NonZero types, there is no way zero can be the result of any
299                // atomic operation. Thus this should never happen.
300                Self::new(src).expect("zero value in `Atom::unpack` for NonZero type")
301            }
302        }
303    };
304    (@int_methods $ty:ty, $non_zero_ty:ident, $impl_ty:ident, false) => {};
305}
306
307#[cfg(target_has_atomic = "8")] impl_std_atomics!(bool, _Dummy, AtomicBool, false);
308#[cfg(target_has_atomic = "8")] impl_std_atomics!(u8, NonZeroU8, AtomicU8, true);
309#[cfg(target_has_atomic = "8")] impl_std_atomics!(i8, NonZeroI8, AtomicI8, true);
310#[cfg(target_has_atomic = "16")] impl_std_atomics!(u16, NonZeroU16, AtomicU16, true);
311#[cfg(target_has_atomic = "16")] impl_std_atomics!(i16, NonZeroI16, AtomicI16, true);
312#[cfg(target_has_atomic = "32")] impl_std_atomics!(u32, NonZeroU32, AtomicU32, true);
313#[cfg(target_has_atomic = "32")] impl_std_atomics!(i32, NonZeroI32, AtomicI32, true);
314#[cfg(target_has_atomic = "64")] impl_std_atomics!(u64, NonZeroU64, AtomicU64, true);
315#[cfg(target_has_atomic = "64")] impl_std_atomics!(i64, NonZeroI64, AtomicI64, true);
316#[cfg(target_has_atomic = "ptr")] impl_std_atomics!(usize, NonZeroUsize, AtomicUsize, true);
317#[cfg(target_has_atomic = "ptr")] impl_std_atomics!(isize, NonZeroIsize, AtomicIsize, true);
318
319// ----- Implementations for non-atomic primitive types ------------------------------------------
320#[cfg(target_has_atomic = "32")]
321impl Atom for f32 {
322    type Repr = u32;
323    fn pack(self) -> Self::Repr {
324        self.to_bits()
325    }
326    fn unpack(src: Self::Repr) -> Self {
327        Self::from_bits(src)
328    }
329}
330
331#[cfg(target_has_atomic = "64")]
332impl Atom for f64 {
333    type Repr = u64;
334    fn pack(self) -> Self::Repr {
335        self.to_bits()
336    }
337    fn unpack(src: Self::Repr) -> Self {
338        Self::from_bits(src)
339    }
340}
341
342#[cfg(target_has_atomic = "32")]
343impl Atom for char {
344    type Repr = u32;
345    fn pack(self) -> Self::Repr {
346        self.into()
347    }
348    fn unpack(src: Self::Repr) -> Self {
349        Self::try_from(src).expect("invalid value in <char as Atom>::unpack")
350    }
351}
352
353// We do not implement `AtomInteger` as, to me, it seems like the exact adding
354// and subtraction behavior of integer atomics is not defined anywhere.
355impl<T: Atom> Atom for Wrapping<T> {
356    type Repr = T::Repr;
357    fn pack(self) -> Self::Repr {
358        self.0.pack()
359    }
360    fn unpack(src: Self::Repr) -> Self {
361        Self(T::unpack(src))
362    }
363}
364impl<T: AtomLogic> AtomLogic for Wrapping<T> where T::Repr: PrimitiveAtomLogic {}
365
366
367#[cfg(target_has_atomic = "ptr")]
368impl<T> Atom for core::ptr::NonNull<T> {
369    type Repr = *mut T;
370    fn pack(self) -> Self::Repr {
371        self.as_ptr().pack()
372    }
373    fn unpack(src: Self::Repr) -> Self {
374        Self::new(<*mut T>::unpack(src))
375            .expect("null value in `<NonNull<T> as Atom>::unpack`")
376    }
377}
378
379#[cfg(target_has_atomic = "ptr")]
380impl<T> Atom for Option<core::ptr::NonNull<T>> {
381    type Repr = *mut T;
382    fn pack(self) -> Self::Repr {
383        self.map(|nn| nn.as_ptr())
384            .unwrap_or(core::ptr::null_mut())
385            .pack()
386    }
387    fn unpack(src: Self::Repr) -> Self {
388        if src.is_null() {
389            None
390        } else {
391            Some(core::ptr::NonNull::new(<*mut T>::unpack(src)).unwrap())
392        }
393    }
394}
395
396macro_rules! impl_option_non_zero {
397    ($ty:ident = $repr:ty) => {
398        impl Atom for Option<core::num::$ty> {
399            type Repr = $repr;
400            fn pack(self) -> Self::Repr {
401                self.map(core::num::$ty::get).unwrap_or(0).pack()
402            }
403            fn unpack(src: Self::Repr) -> Self {
404                core::num::$ty::new(src)
405            }
406        }
407
408        // Semantically, an `Option<NonZeroFoo>` represents `Foo` exactly. It
409        // also has the exact same memory layout. It's just that we assign
410        // the "symbol" `None` to 0. Any integer operation that leads to 0 on
411        // the underlying type will result in `None`.
412        impl AtomInteger for Option<core::num::$ty> {}
413    };
414}
415
416#[cfg(target_has_atomic = "8")] impl_option_non_zero!(NonZeroU8 = u8);
417#[cfg(target_has_atomic = "8")] impl_option_non_zero!(NonZeroI8 = i8);
418#[cfg(target_has_atomic = "16")] impl_option_non_zero!(NonZeroU16 = u16);
419#[cfg(target_has_atomic = "16")] impl_option_non_zero!(NonZeroI16 = i16);
420#[cfg(target_has_atomic = "32")] impl_option_non_zero!(NonZeroU32 = u32);
421#[cfg(target_has_atomic = "32")] impl_option_non_zero!(NonZeroI32 = i32);
422#[cfg(target_has_atomic = "64")] impl_option_non_zero!(NonZeroU64 = u64);
423#[cfg(target_has_atomic = "64")] impl_option_non_zero!(NonZeroI64 = i64);
424#[cfg(target_has_atomic = "ptr")] impl_option_non_zero!(NonZeroUsize = usize);
425#[cfg(target_has_atomic = "ptr")] impl_option_non_zero!(NonZeroIsize = isize);
426
427macro_rules! impl_int8_arrays {
428    ($elem:ident, $len:literal, $repr:ident) => {
429        impl Atom for [$elem; $len] {
430            type Repr = $repr;
431            fn pack(self) -> Self::Repr {
432                $repr::from_ne_bytes(self.map(|e| e as _))
433            }
434            fn unpack(src: Self::Repr) -> Self {
435                src.to_ne_bytes().map(|e| e as _)
436            }
437        }
438        impl AtomLogic for [$elem; $len] {}
439    };
440}
441
442#[cfg(target_has_atomic = "16")] impl_int8_arrays!(u8, 2, u16);
443#[cfg(target_has_atomic = "32")] impl_int8_arrays!(u8, 4, u32);
444#[cfg(target_has_atomic = "64")] impl_int8_arrays!(u8, 8, u64);
445#[cfg(target_has_atomic = "16")] impl_int8_arrays!(i8, 2, u16);
446#[cfg(target_has_atomic = "32")] impl_int8_arrays!(i8, 4, u32);
447#[cfg(target_has_atomic = "64")] impl_int8_arrays!(i8, 8, u64);
448
449macro_rules! impl_int_arrays {
450    ($unsigned_elem:ident, $signed_elem:ident, $len:literal, $repr:ident, $nested:tt, $flat:tt) => {
451        impl_small_primitive_array!($unsigned_elem, $len, $repr, $nested, $flat);
452        impl_small_primitive_array!($signed_elem, $len, $repr, $nested, $flat);
453    };
454}
455
456macro_rules! impl_small_primitive_array {
457    ($elem:ident, $len:literal, $repr:ident, $nested:tt, $flat:tt) => {
458        impl Atom for [$elem; $len] {
459            type Repr = $repr;
460            fn pack(self) -> Self::Repr {
461                let $nested = self.map(|x| x.to_ne_bytes());
462                Self::Repr::from_ne_bytes($flat)
463            }
464            fn unpack(src: Self::Repr) -> Self {
465                let $flat = src.to_ne_bytes();
466                $nested.map($elem::from_ne_bytes)
467            }
468        }
469        impl AtomLogic for [$elem; $len] {}
470    };
471}
472#[cfg(target_has_atomic = "32")] impl_int_arrays!(u16, i16, 2, u32,
473    [[b0, b1], [b2, b3]], [b0, b1, b2, b3]);
474#[cfg(target_has_atomic = "64")] impl_int_arrays!(u16, i16, 4, u64,
475    [[b0, b1], [b2, b3], [b4, b5], [b6, b7]], [b0, b1, b2, b3, b4, b5, b6, b7]);
476#[cfg(target_has_atomic = "64")] impl_int_arrays!(u32, i32, 2, u64,
477    [[b0, b1, b2, b3], [b4, b5, b6, b7]], [b0, b1, b2, b3, b4, b5, b6, b7]);
478
479/// This is just a dummy module to have doc tests.
480///
481/// ```
482/// use atomig::{Atom, AtomLogic, AtomInteger, impls::{PrimitiveAtomLogic, PrimitiveAtomInteger}};
483///
484/// fn assert_impl_atom<T: Atom>() {}
485/// fn assert_impl_atom_logic<T: AtomLogic>()
486/// where
487///     T::Repr: PrimitiveAtomLogic,
488/// {}
489/// fn assert_impl_atom_all<T: AtomLogic + AtomInteger>()
490/// where
491///     T::Repr: PrimitiveAtomInteger + PrimitiveAtomLogic,
492/// {}
493///
494/// assert_impl_atom_all::<u8>();
495/// assert_impl_atom_all::<i8>();
496/// assert_impl_atom_all::<u16>();
497/// assert_impl_atom_all::<i16>();
498/// assert_impl_atom_all::<u32>();
499/// assert_impl_atom_all::<i32>();
500/// assert_impl_atom_all::<u64>();
501/// assert_impl_atom_all::<i64>();
502/// assert_impl_atom_all::<usize>();
503/// assert_impl_atom_all::<isize>();
504///
505/// assert_impl_atom_logic::<bool>();
506///
507/// assert_impl_atom::<*mut ()>();
508/// assert_impl_atom::<*mut String>();
509/// assert_impl_atom::<core::ptr::NonNull<()>>();
510/// assert_impl_atom::<core::ptr::NonNull<String>>();
511/// assert_impl_atom::<Option<core::ptr::NonNull<()>>>();
512/// assert_impl_atom::<Option<core::ptr::NonNull<String>>>();
513///
514/// assert_impl_atom::<char>();
515/// assert_impl_atom::<f32>();
516/// assert_impl_atom::<f64>();
517///
518/// assert_impl_atom::<[u8; 2]>();
519/// assert_impl_atom::<[u8; 4]>();
520/// assert_impl_atom::<[u8; 8]>();
521/// assert_impl_atom::<[i8; 2]>();
522/// assert_impl_atom::<[i8; 4]>();
523/// assert_impl_atom::<[i8; 8]>();
524/// assert_impl_atom::<[u16; 2]>();
525/// assert_impl_atom::<[u16; 4]>();
526/// assert_impl_atom::<[i16; 2]>();
527/// assert_impl_atom::<[i16; 4]>();
528/// assert_impl_atom::<[u32; 2]>();
529/// assert_impl_atom::<[i32; 2]>();
530///
531/// assert_impl_atom::<core::num::NonZeroU8>();
532/// assert_impl_atom::<core::num::NonZeroI8>();
533/// assert_impl_atom::<core::num::NonZeroU16>();
534/// assert_impl_atom::<core::num::NonZeroI16>();
535/// assert_impl_atom::<core::num::NonZeroU32>();
536/// assert_impl_atom::<core::num::NonZeroI32>();
537/// assert_impl_atom::<core::num::NonZeroU64>();
538/// assert_impl_atom::<core::num::NonZeroI64>();
539/// assert_impl_atom::<core::num::NonZeroUsize>();
540/// assert_impl_atom::<core::num::NonZeroIsize>();
541///
542/// assert_impl_atom_logic::<core::num::Wrapping<u8>>();
543/// assert_impl_atom_logic::<core::num::Wrapping<i8>>();
544/// assert_impl_atom_logic::<core::num::Wrapping<u16>>();
545/// assert_impl_atom_logic::<core::num::Wrapping<i16>>();
546/// assert_impl_atom_logic::<core::num::Wrapping<u32>>();
547/// assert_impl_atom_logic::<core::num::Wrapping<i32>>();
548/// assert_impl_atom_logic::<core::num::Wrapping<u64>>();
549/// assert_impl_atom_logic::<core::num::Wrapping<i64>>();
550/// assert_impl_atom_logic::<core::num::Wrapping<usize>>();
551/// assert_impl_atom_logic::<core::num::Wrapping<isize>>();
552/// ```
553mod tests {}
atomig/impls.rs

atomig/
impls.rs
