atomic_maybe_uninit/

raw.rs

// SPDX-License-Identifier: Apache-2.0 OR MIT

//! Low level API.

#[cfg(doc)]
use core::{
    cell::UnsafeCell,
    sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst},
};
use core::{mem::MaybeUninit, sync::atomic::Ordering};

// TODO(semver): merge AtomicLoad and AtomicStore and rename to AtomicLoadStore?

/// Primitive types that may support atomic operations.
///
/// This trait is sealed and cannot be implemented for types outside of `atomic-maybe-uninit`.
///
/// Currently this is implemented only for integer types.
pub trait Primitive: crate::private::PrimitivePriv {}

/// Atomic load.
///
/// This trait is sealed and cannot be implemented for types outside of `atomic-maybe-uninit`.
#[cfg_attr(
    not(atomic_maybe_uninit_no_diagnostic_namespace),
    diagnostic::on_unimplemented(
        message = "atomic load of `{Self}` is not available on this target",
        label = "this associated function is not available on this target",
        note = "see <https://docs.rs/atomic-maybe-uninit/latest/atomic_maybe_uninit/#platform-support> for more."
    )
)]
pub trait AtomicLoad: Primitive {
    /// Loads a value from `src`.
    ///
    /// `atomic_load` takes an [`Ordering`] argument which describes the memory ordering of this operation.
    /// Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`].
    ///
    /// # Safety
    ///
    /// Behavior is undefined if any of the following conditions are violated:
    ///
    /// - `src` must be [valid] for reads.
    /// - `src` must be aligned to `size_of::<MaybeUninit<T>>()` (note that on some platforms this
    ///   can be bigger than `align_of::<MaybeUninit<T>>()`).
    /// - `order` must be [`SeqCst`], [`Acquire`], or [`Relaxed`].
    /// - You must adhere to the [Memory model for atomic accesses]. In particular, it is not
    ///   allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different
    ///   sizes, without synchronization.
    ///
    /// Compatibility with read-only memory applies only to relaxed operations with a register or smaller width.
    /// See the ["Atomic accesses to read-only memory" section in the `core::sync::atomic` docs][read-only-memory]
    /// for more.
    ///
    /// [valid]: core::ptr#safety
    /// [Memory model for atomic accesses]: core::sync::atomic#memory-model-for-atomic-accesses
    /// [read-only-memory]: core::sync::atomic#atomic-accesses-to-read-only-memory
    unsafe fn atomic_load(src: *const MaybeUninit<Self>, order: Ordering) -> MaybeUninit<Self>;
}

/// Atomic store.
///
/// This trait is sealed and cannot be implemented for types outside of `atomic-maybe-uninit`.
#[cfg_attr(
    not(atomic_maybe_uninit_no_diagnostic_namespace),
    diagnostic::on_unimplemented(
        message = "atomic store of `{Self}` is not available on this target",
        label = "this associated function is not available on this target",
        note = "see <https://docs.rs/atomic-maybe-uninit/latest/atomic_maybe_uninit/#platform-support> for more."
    )
)]
pub trait AtomicStore: Primitive {
    /// Stores a value into `dst`.
    ///
    /// `atomic_store` takes an [`Ordering`] argument which describes the memory ordering of this operation.
    ///  Possible values are [`SeqCst`], [`Release`] and [`Relaxed`].
    ///
    /// # Safety
    ///
    /// Behavior is undefined if any of the following conditions are violated:
    ///
    /// - `dst` must be [valid] for writes
    /// - `dst` must be aligned to `size_of::<MaybeUninit<T>>()` (note that on some platforms this
    ///   can be bigger than `align_of::<MaybeUninit<T>>()`).
    /// - `order` must be [`SeqCst`], [`Release`], or [`Relaxed`].
    /// - You must adhere to the [Memory model for atomic accesses]. In particular, it is not
    ///   allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different
    ///   sizes, without synchronization.
    ///
    /// Compatibility with write-only memory applies only to relaxed operations with a register or smaller width.
    /// See the ["Atomic accesses to read-only memory" section in the `core::sync::atomic` docs][read-only-memory]
    /// for more.
    ///
    /// [valid]: core::ptr#safety
    /// [Memory model for atomic accesses]: core::sync::atomic#memory-model-for-atomic-accesses
    /// [read-only-memory]: core::sync::atomic#atomic-accesses-to-read-only-memory
    #[inline]
    unsafe fn atomic_store(dst: *mut MaybeUninit<Self>, val: MaybeUninit<Self>, order: Ordering) {
        // Workaround LLVM pre-20 bug: https://github.com/rust-lang/rust/issues/129585#issuecomment-2360273081
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let val = core::hint::black_box(val);
        // SAFETY: the caller must uphold the safety contract.
        unsafe { Self::__atomic_store_impl(dst, val, order) }
    }

    #[doc(hidden)] // Not public API.
    unsafe fn __atomic_store_impl(
        dst: *mut MaybeUninit<Self>,
        val: MaybeUninit<Self>,
        order: Ordering,
    );
}

/// Atomic swap.
///
/// This trait is sealed and cannot be implemented for types outside of `atomic-maybe-uninit`.
#[cfg_attr(
    not(atomic_maybe_uninit_no_diagnostic_namespace),
    diagnostic::on_unimplemented(
        message = "atomic swap of `{Self}` is not available on this target",
        label = "this associated function is not available on this target",
        note = "see <https://docs.rs/atomic-maybe-uninit/latest/atomic_maybe_uninit/#platform-support> for more."
    )
)]
pub trait AtomicSwap: AtomicLoad + AtomicStore {
    /// Stores a value into `dst`, returning the previous value.
    ///
    /// `atomic_swap` takes an [`Ordering`] argument which describes the memory ordering
    /// of this operation. All ordering modes are possible. Note that using
    /// [`Acquire`] makes the store part of this operation [`Relaxed`], and
    /// using [`Release`] makes the load part [`Relaxed`].
    ///
    /// # Safety
    ///
    /// Behavior is undefined if any of the following conditions are violated:
    ///
    /// - `dst` must be [valid] for both reads and writes.
    /// - `dst` must be aligned to `size_of::<MaybeUninit<T>>()` (note that on some platforms this
    ///   can be bigger than `align_of::<MaybeUninit<T>>()`).
    /// - `order` must be [`SeqCst`], [`AcqRel`], [`Acquire`], [`Release`], or [`Relaxed`].
    /// - You must adhere to the [Memory model for atomic accesses]. In particular, it is not
    ///   allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different
    ///   sizes, without synchronization.
    ///
    /// [valid]: core::ptr#safety
    /// [Memory model for atomic accesses]: core::sync::atomic#memory-model-for-atomic-accesses
    #[inline]
    unsafe fn atomic_swap(
        dst: *mut MaybeUninit<Self>,
        val: MaybeUninit<Self>,
        order: Ordering,
    ) -> MaybeUninit<Self> {
        // Workaround LLVM pre-20 bug: https://github.com/rust-lang/rust/issues/129585#issuecomment-2360273081
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let val = core::hint::black_box(val);
        // SAFETY: the caller must uphold the safety contract.
        unsafe { Self::__atomic_swap_impl(dst, val, order) }
    }

    #[doc(hidden)] // Not public API.
    unsafe fn __atomic_swap_impl(
        dst: *mut MaybeUninit<Self>,
        val: MaybeUninit<Self>,
        order: Ordering,
    ) -> MaybeUninit<Self>;
}

/// Atomic compare and exchange.
///
/// This trait is sealed and cannot be implemented for types outside of `atomic-maybe-uninit`.
#[cfg_attr(
    not(atomic_maybe_uninit_no_diagnostic_namespace),
    diagnostic::on_unimplemented(
        message = "atomic compare and exchange of `{Self}` is not available on this target",
        label = "this associated function is not available on this target",
        note = "see <https://docs.rs/atomic-maybe-uninit/latest/atomic_maybe_uninit/#platform-support> for more."
    )
)]
pub trait AtomicCompareExchange: AtomicLoad + AtomicStore {
    /// Stores a value into `dst` if the current value is the same as
    /// the `current` value. Here, "the same" is determined using byte-wise
    /// equality, not `PartialEq`.
    ///
    /// The return value is a tuple of the previous value and the result indicating whether the new
    /// value was written and containing the previous value. On success, the returned value is
    /// guaranteed to be equal to the value at `current`.
    ///
    /// `atomic_compare_exchange` takes two [`Ordering`] arguments to describe the memory
    /// ordering of this operation. `success` describes the required ordering for the
    /// read-modify-write operation that takes place if the comparison with `current` succeeds.
    /// `failure` describes the required ordering for the load operation that takes place when
    /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
    /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
    ///
    /// # Safety
    ///
    /// Behavior is undefined if any of the following conditions are violated:
    ///
    /// - `dst` must be valid for both reads and writes.
    /// - `dst` must be aligned to `size_of::<MaybeUninit<T>>()` (note that on some platforms this
    ///   can be bigger than `align_of::<MaybeUninit<T>>()`).
    /// - `success` must be [`SeqCst`], [`AcqRel`], [`Acquire`], [`Release`], or [`Relaxed`].
    /// - `failure` must be [`SeqCst`], [`Acquire`], or [`Relaxed`].
    /// - You must adhere to the [Memory model for atomic accesses]. In particular, it is not
    ///   allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different
    ///   sizes, without synchronization.
    ///
    /// [valid]: core::ptr#safety
    /// [Memory model for atomic accesses]: core::sync::atomic#memory-model-for-atomic-accesses
    ///
    /// # Notes
    ///
    /// Comparison of two values containing uninitialized bytes may fail even if
    /// they are equivalent as Rust's type, because values can be byte-wise
    /// inequal even when they are equal as Rust values.
    ///
    /// See [`AtomicMaybeUninit::compare_exchange`](crate::AtomicMaybeUninit::compare_exchange) for details.
    #[inline]
    unsafe fn atomic_compare_exchange(
        dst: *mut MaybeUninit<Self>,
        current: MaybeUninit<Self>,
        new: MaybeUninit<Self>,
        success: Ordering,
        failure: Ordering,
    ) -> (MaybeUninit<Self>, bool) {
        // Workaround LLVM pre-20 bug: https://github.com/rust-lang/rust/issues/129585#issuecomment-2360273081
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let current = core::hint::black_box(current);
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let new = core::hint::black_box(new);
        // SAFETY: the caller must uphold the safety contract.
        unsafe { Self::__atomic_compare_exchange_impl(dst, current, new, success, failure) }
    }

    #[doc(hidden)] // Not public API.
    unsafe fn __atomic_compare_exchange_impl(
        dst: *mut MaybeUninit<Self>,
        current: MaybeUninit<Self>,
        new: MaybeUninit<Self>,
        success: Ordering,
        failure: Ordering,
    ) -> (MaybeUninit<Self>, bool);

    /// Stores a value into `dst` if the current value is the same as
    /// the `current` value. Here, "the same" is determined using byte-wise
    /// equality, not `PartialEq`.
    ///
    /// This function is allowed to spuriously fail even when the comparison succeeds, which can
    /// result in more efficient code on some platforms. The return value is a tuple of the previous
    /// value and the result indicating whether the new value was written and containing the
    /// previous value.
    ///
    /// `atomic_compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory
    /// ordering of this operation. `success` describes the required ordering for the
    /// read-modify-write operation that takes place if the comparison with `current` succeeds.
    /// `failure` describes the required ordering for the load operation that takes place when
    /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
    /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
    ///
    /// # Safety
    ///
    /// Behavior is undefined if any of the following conditions are violated:
    ///
    /// - `dst` must be [valid] for both reads and writes.
    /// - `dst` must be aligned to `size_of::<MaybeUninit<T>>()` (note that on some platforms this
    ///   can be bigger than `align_of::<MaybeUninit<T>>()`).
    /// - `success` must be [`SeqCst`], [`AcqRel`], [`Acquire`], [`Release`], or [`Relaxed`].
    /// - `failure` must be [`SeqCst`], [`Acquire`], or [`Relaxed`].
    /// - You must adhere to the [Memory model for atomic accesses]. In particular, it is not
    ///   allowed to mix conflicting atomic and non-atomic accesses, or atomic accesses of different
    ///   sizes, without synchronization.
    ///
    /// [valid]: core::ptr#safety
    /// [Memory model for atomic accesses]: core::sync::atomic#memory-model-for-atomic-accesses
    ///
    /// # Notes
    ///
    /// Comparison of two values containing uninitialized bytes may fail even if
    /// they are equivalent as Rust's type, because values can be byte-wise
    /// inequal even when they are equal as Rust values.
    ///
    /// See [`AtomicMaybeUninit::compare_exchange`](crate::AtomicMaybeUninit::compare_exchange) for details.
    #[inline]
    unsafe fn atomic_compare_exchange_weak(
        dst: *mut MaybeUninit<Self>,
        current: MaybeUninit<Self>,
        new: MaybeUninit<Self>,
        success: Ordering,
        failure: Ordering,
    ) -> (MaybeUninit<Self>, bool) {
        // Workaround LLVM pre-20 bug: https://github.com/rust-lang/rust/issues/129585#issuecomment-2360273081
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let current = core::hint::black_box(current);
        #[cfg(not(atomic_maybe_uninit_llvm_20_or_later))]
        let new = core::hint::black_box(new);
        // SAFETY: the caller must uphold the safety contract.
        unsafe { Self::__atomic_compare_exchange_weak_impl(dst, current, new, success, failure) }
    }

    #[doc(hidden)] // Not public API.
    #[inline]
    unsafe fn __atomic_compare_exchange_weak_impl(
        dst: *mut MaybeUninit<Self>,
        current: MaybeUninit<Self>,
        new: MaybeUninit<Self>,
        success: Ordering,
        failure: Ordering,
    ) -> (MaybeUninit<Self>, bool) {
        // SAFETY: the caller must uphold the safety contract.
        unsafe { Self::__atomic_compare_exchange_impl(dst, current, new, success, failure) }
    }
}