portable-atomic 1.6.0

// SPDX-License-Identifier: Apache-2.0 OR MIT

// Atomic{I,U}128 implementation on x86_64 using CMPXCHG16B (DWCAS).
//
// Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
// this module and use intrinsics.rs instead.
//
// Refs:
// - x86 and amd64 instruction reference https://www.felixcloutier.com/x86
// - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit
//
// Generated asm:
// - x86_64 (+cmpxchg16b) https://godbolt.org/z/55n54WeKr

include!("macros.rs");

#[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
#[path = "../fallback/outline_atomics.rs"]
mod fallback;

#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(not(target_env = "sgx"))]
#[path = "detect/x86_64.rs"]
mod detect;

#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::sync::atomic::Ordering;

use crate::utils::{Pair, U128};

// Asserts that the function is called in the correct context.
macro_rules! debug_assert_cmpxchg16b {
    () => {
        #[cfg(not(any(
            target_feature = "cmpxchg16b",
            portable_atomic_target_feature = "cmpxchg16b",
        )))]
        {
            debug_assert!(detect::detect().has_cmpxchg16b());
        }
    };
}
#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))]
#[cfg(target_feature = "sse")]
macro_rules! debug_assert_vmovdqa_atomic {
    () => {{
        debug_assert_cmpxchg16b!();
        debug_assert!(detect::detect().has_vmovdqa_atomic());
    }};
}

#[allow(unused_macros)]
#[cfg(target_pointer_width = "32")]
macro_rules! ptr_modifier {
    () => {
        ":e"
    };
}
#[allow(unused_macros)]
#[cfg(target_pointer_width = "64")]
macro_rules! ptr_modifier {
    () => {
        ""
    };
}

#[cfg_attr(
    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
    target_feature(enable = "cmpxchg16b")
)]
#[inline]
unsafe fn cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
    debug_assert!(dst as usize % 16 == 0);
    debug_assert_cmpxchg16b!();

    // SAFETY: the caller must guarantee that `dst` is valid for both writes and
    // reads, 16-byte aligned (required by CMPXCHG16B), that there are no
    // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.
    //
    // If the value at `dst` (destination operand) and rdx:rax are equal, the
    // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at
    // `dst` is loaded to rdx:rax.
    //
    // The ZF flag is set if the value at `dst` and rdx:rax are equal,
    // otherwise it is cleared. Other flags are unaffected.
    //
    // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
    unsafe {
        // cmpxchg16b is always SeqCst.
        let r: u8;
        let old = U128 { whole: old };
        let new = U128 { whole: new };
        let (prev_lo, prev_hi);
        macro_rules! cmpxchg16b {
            ($rdi:tt) => {
                asm!(
                    // rbx is reserved by LLVM
                    "xchg {rbx_tmp}, rbx",
                    concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
                    "sete r8b",
                    "mov rbx, {rbx_tmp}", // restore rbx
                    rbx_tmp = inout(reg) new.pair.lo => _,
                    in("rcx") new.pair.hi,
                    inout("rax") old.pair.lo => prev_lo,
                    inout("rdx") old.pair.hi => prev_hi,
                    in($rdi) dst,
                    out("r8b") r,
                    // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
                    options(nostack),
                )
            };
        }
        #[cfg(target_pointer_width = "32")]
        cmpxchg16b!("edi");
        #[cfg(target_pointer_width = "64")]
        cmpxchg16b!("rdi");
        (U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole, r != 0)
    }
}

// VMOVDQA is atomic on Intel and AMD CPUs with AVX.
// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
//
// Refs: https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64
//
// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))]
#[cfg(target_feature = "sse")]
#[target_feature(enable = "avx")]
#[inline]
unsafe fn atomic_load_vmovdqa(src: *mut u128) -> u128 {
    debug_assert!(src as usize % 16 == 0);
    debug_assert_vmovdqa_atomic!();

    // SAFETY: the caller must uphold the safety contract.
    //
    // atomic load by vmovdqa is always SeqCst.
    unsafe {
        let out: core::arch::x86_64::__m128;
        asm!(
            concat!("vmovdqa {out}, xmmword ptr [{src", ptr_modifier!(), "}]"),
            src = in(reg) src,
            out = out(xmm_reg) out,
            options(nostack, preserves_flags),
        );
        core::mem::transmute(out)
    }
}
#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))]
#[cfg(target_feature = "sse")]
#[target_feature(enable = "avx")]
#[inline]
unsafe fn atomic_store_vmovdqa(dst: *mut u128, val: u128, order: Ordering) {
    debug_assert!(dst as usize % 16 == 0);
    debug_assert_vmovdqa_atomic!();

    // SAFETY: the caller must uphold the safety contract.
    unsafe {
        let val: core::arch::x86_64::__m128 = core::mem::transmute(val);
        match order {
            // Relaxed and Release stores are equivalent.
            Ordering::Relaxed | Ordering::Release => {
                asm!(
                    concat!("vmovdqa xmmword ptr [{dst", ptr_modifier!(), "}], {val}"),
                    dst = in(reg) dst,
                    val = in(xmm_reg) val,
                    options(nostack, preserves_flags),
                );
            }
            Ordering::SeqCst => {
                asm!(
                    concat!("vmovdqa xmmword ptr [{dst", ptr_modifier!(), "}], {val}"),
                    "mfence",
                    dst = in(reg) dst,
                    val = in(xmm_reg) val,
                    options(nostack, preserves_flags),
                );
            }
            _ => unreachable!("{:?}", order),
        }
    }
}

#[cfg(not(all(
    any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
    any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")),
)))]
macro_rules! load_store_detect {
    (
        vmovdqa = $vmovdqa:ident
        cmpxchg16b = $cmpxchg16b:ident
        fallback = $fallback:ident
    ) => {{
        let cpuid = detect::detect();
        #[cfg(not(any(
            target_feature = "cmpxchg16b",
            portable_atomic_target_feature = "cmpxchg16b",
        )))]
        {
            // Check CMPXCHG16B first to prevent mixing atomic and non-atomic access.
            if cpuid.has_cmpxchg16b() {
                // We do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
                #[cfg(target_feature = "sse")]
                {
                    if cpuid.has_vmovdqa_atomic() {
                        $vmovdqa
                    } else {
                        $cmpxchg16b
                    }
                }
                #[cfg(not(target_feature = "sse"))]
                {
                    $cmpxchg16b
                }
            } else {
                fallback::$fallback
            }
        }
        #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
        {
            if cpuid.has_vmovdqa_atomic() {
                $vmovdqa
            } else {
                $cmpxchg16b
            }
        }
    }};
}

#[inline]
unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {
    // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
    // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
    // SGX doesn't support CPUID.
    #[cfg(all(
        any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
        any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")),
    ))]
    // SAFETY: the caller must uphold the safety contract.
    // cfg guarantees that CMPXCHG16B is available at compile-time.
    unsafe {
        // cmpxchg16b is always SeqCst.
        atomic_load_cmpxchg16b(src)
    }
    #[cfg(not(all(
        any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
        any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")),
    )))]
    // SAFETY: the caller must uphold the safety contract.
    unsafe {
        ifunc!(unsafe fn(src: *mut u128) -> u128 {
            load_store_detect! {
                vmovdqa = atomic_load_vmovdqa
                cmpxchg16b = atomic_load_cmpxchg16b
                // Use SeqCst because cmpxchg16b and atomic load by vmovdqa is always SeqCst.
                fallback = atomic_load_seqcst
            }
        })
    }
}
#[cfg_attr(
    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
    target_feature(enable = "cmpxchg16b")
)]
#[inline]
unsafe fn atomic_load_cmpxchg16b(src: *mut u128) -> u128 {
    debug_assert!(src as usize % 16 == 0);
    debug_assert_cmpxchg16b!();

    // SAFETY: the caller must guarantee that `src` is valid for both writes and
    // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
    // cfg guarantees that the CPU supports CMPXCHG16B.
    //
    // See cmpxchg16b function for more.
    //
    // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows
    // omitting the storing of condition flags and avoid use of xchg to handle rbx.
    unsafe {
        // cmpxchg16b is always SeqCst.
        let (out_lo, out_hi);
        macro_rules! cmpxchg16b {
            ($rdi:tt) => {
                asm!(
                    // rbx is reserved by LLVM
                    "mov {rbx_tmp}, rbx",
                    "xor rbx, rbx", // zeroed rbx
                    concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
                    "mov rbx, {rbx_tmp}", // restore rbx
                    // set old/new args of cmpxchg16b to 0 (rbx is zeroed after saved to rbx_tmp, to avoid xchg)
                    rbx_tmp = out(reg) _,
                    in("rcx") 0_u64,
                    inout("rax") 0_u64 => out_lo,
                    inout("rdx") 0_u64 => out_hi,
                    in($rdi) src,
                    // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
                    options(nostack),
                )
            };
        }
        #[cfg(target_pointer_width = "32")]
        cmpxchg16b!("edi");
        #[cfg(target_pointer_width = "64")]
        cmpxchg16b!("rdi");
        U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
    }
}

#[inline]
unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
    // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
    // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
    // SGX doesn't support CPUID.
    #[cfg(all(
        any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
        any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")),
    ))]
    // SAFETY: the caller must uphold the safety contract.
    // cfg guarantees that CMPXCHG16B is available at compile-time.
    unsafe {
        // cmpxchg16b is always SeqCst.
        let _ = order;
        atomic_store_cmpxchg16b(dst, val);
    }
    #[cfg(not(all(
        any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
        any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")),
    )))]
    // SAFETY: the caller must uphold the safety contract.
    unsafe {
        #[cfg(target_feature = "sse")]
        fn_alias! {
            #[target_feature(enable = "avx")]
            unsafe fn(dst: *mut u128, val: u128);
            // atomic store by vmovdqa has at least release semantics.
            atomic_store_vmovdqa_non_seqcst = atomic_store_vmovdqa(Ordering::Release);
            atomic_store_vmovdqa_seqcst = atomic_store_vmovdqa(Ordering::SeqCst);
        }
        match order {
            // Relaxed and Release stores are equivalent in all implementations
            // that may be called here (vmovdqa, asm-based cmpxchg16b, and fallback).
            // core::arch's cmpxchg16b will never called here.
            Ordering::Relaxed | Ordering::Release => {
                ifunc!(unsafe fn(dst: *mut u128, val: u128) {
                    load_store_detect! {
                        vmovdqa = atomic_store_vmovdqa_non_seqcst
                        cmpxchg16b = atomic_store_cmpxchg16b
                        fallback = atomic_store_non_seqcst
                    }
                });
            }
            Ordering::SeqCst => {
                ifunc!(unsafe fn(dst: *mut u128, val: u128) {
                    load_store_detect! {
                        vmovdqa = atomic_store_vmovdqa_seqcst
                        cmpxchg16b = atomic_store_cmpxchg16b
                        fallback = atomic_store_seqcst
                    }
                });
            }
            _ => unreachable!("{:?}", order),
        }
    }
}
#[cfg_attr(
    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
    target_feature(enable = "cmpxchg16b")
)]
unsafe fn atomic_store_cmpxchg16b(dst: *mut u128, val: u128) {
    // SAFETY: the caller must uphold the safety contract.
    unsafe {
        // cmpxchg16b is always SeqCst.
        atomic_swap_cmpxchg16b(dst, val, Ordering::SeqCst);
    }
}

#[inline]
unsafe fn atomic_compare_exchange(
    dst: *mut u128,
    old: u128,
    new: u128,
    _success: Ordering,
    _failure: Ordering,
) -> Result<u128, u128> {
    #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
    // SAFETY: the caller must guarantee that `dst` is valid for both writes and
    // reads, 16-byte aligned, that there are no concurrent non-atomic operations,
    // and cfg guarantees that CMPXCHG16B is available at compile-time.
    let (prev, ok) = unsafe { cmpxchg16b(dst, old, new) };
    #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
    // SAFETY: the caller must guarantee that `dst` is valid for both writes and
    // reads, 16-byte aligned, and that there are no different kinds of concurrent accesses.
    let (prev, ok) = unsafe {
        ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
            if detect::detect().has_cmpxchg16b() {
                cmpxchg16b
            } else {
                // Use SeqCst because cmpxchg16b is always SeqCst.
                fallback::atomic_compare_exchange_seqcst
            }
        })
    };
    if ok {
        Ok(prev)
    } else {
        Err(prev)
    }
}

// cmpxchg16b is always strong.
use atomic_compare_exchange as atomic_compare_exchange_weak;

#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
use atomic_swap_cmpxchg16b as atomic_swap;
#[cfg_attr(
    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
    target_feature(enable = "cmpxchg16b")
)]
#[inline]
unsafe fn atomic_swap_cmpxchg16b(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
    debug_assert!(dst as usize % 16 == 0);
    debug_assert_cmpxchg16b!();

    // SAFETY: the caller must guarantee that `dst` is valid for both writes and
    // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
    // cfg guarantees that the CPU supports CMPXCHG16B.
    //
    // See cmpxchg16b function for more.
    //
    // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows
    // omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx.
    //
    // Do not use atomic_rmw_cas_3 because it needs extra MOV to implement swap.
    unsafe {
        // cmpxchg16b is always SeqCst.
        let val = U128 { whole: val };
        let (mut prev_lo, mut prev_hi);
        macro_rules! cmpxchg16b {
            ($rdi:tt) => {
                asm!(
                    // rbx is reserved by LLVM
                    "xchg {rbx_tmp}, rbx",
                    // This is not single-copy atomic reads, but this is ok because subsequent
                    // CAS will check for consistency.
                    //
                    // This is based on the code generated for the first load in DW RMWs by LLVM.
                    //
                    // Note that the C++20 memory model does not allow mixed-sized atomic access,
                    // so we must use inline assembly to implement this.
                    // (i.e., byte-wise atomic based on the standard library's atomic types
                    // cannot be used here).
                    concat!("mov rax, qword ptr [", $rdi, "]"),
                    concat!("mov rdx, qword ptr [", $rdi, " + 8]"),
                    "2:",
                        concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
                        "jne 2b",
                    "mov rbx, {rbx_tmp}", // restore rbx
                    rbx_tmp = inout(reg) val.pair.lo => _,
                    in("rcx") val.pair.hi,
                    out("rax") prev_lo,
                    out("rdx") prev_hi,
                    in($rdi) dst,
                    // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
                    options(nostack),
                )
            };
        }
        #[cfg(target_pointer_width = "32")]
        cmpxchg16b!("edi");
        #[cfg(target_pointer_width = "64")]
        cmpxchg16b!("rdi");
        U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
    }
}

/// Atomic RMW by CAS loop (3 arguments)
/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`
///
/// `$op` can use the following registers:
/// - rsi/r8 pair: val argument (read-only for `$op`)
/// - rax/rdx pair: previous value loaded (read-only for `$op`)
/// - rbx/rcx pair: new value that will be stored
// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows
// omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx.
macro_rules! atomic_rmw_cas_3 {
    ($name:ident as $reexport_name:ident, $($op:tt)*) => {
        #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
        use $name as $reexport_name;
        #[cfg_attr(
            not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
            target_feature(enable = "cmpxchg16b")
        )]
        #[inline]
        unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {
            debug_assert!(dst as usize % 16 == 0);
            debug_assert_cmpxchg16b!();
            // SAFETY: the caller must guarantee that `dst` is valid for both writes and
            // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
            // cfg guarantees that the CPU supports CMPXCHG16B.
            //
            // See cmpxchg16b function for more.
            unsafe {
                // cmpxchg16b is always SeqCst.
                let val = U128 { whole: val };
                let (mut prev_lo, mut prev_hi);
                macro_rules! cmpxchg16b {
                    ($rdi:tt) => {
                        asm!(
                            // rbx is reserved by LLVM
                            "mov {rbx_tmp}, rbx",
                            // This is not single-copy atomic reads, but this is ok because subsequent
                            // CAS will check for consistency.
                            //
                            // This is based on the code generated for the first load in DW RMWs by LLVM.
                            //
                            // Note that the C++20 memory model does not allow mixed-sized atomic access,
                            // so we must use inline assembly to implement this.
                            // (i.e., byte-wise atomic based on the standard library's atomic types
                            // cannot be used here).
                            concat!("mov rax, qword ptr [", $rdi, "]"),
                            concat!("mov rdx, qword ptr [", $rdi, " + 8]"),
                            "2:",
                                $($op)*
                                concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
                                "jne 2b",
                            "mov rbx, {rbx_tmp}", // restore rbx
                            rbx_tmp = out(reg) _,
                            out("rcx") _,
                            out("rax") prev_lo,
                            out("rdx") prev_hi,
                            in($rdi) dst,
                            in("rsi") val.pair.lo,
                            in("r8") val.pair.hi,
                            // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
                            options(nostack),
                        )
                    };
                }
                #[cfg(target_pointer_width = "32")]
                cmpxchg16b!("edi");
                #[cfg(target_pointer_width = "64")]
                cmpxchg16b!("rdi");
                U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
            }
        }
    };
}
/// Atomic RMW by CAS loop (2 arguments)
/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`
///
/// `$op` can use the following registers:
/// - rax/rdx pair: previous value loaded (read-only for `$op`)
/// - rbx/rcx pair: new value that will be stored
// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows
// omitting the storing of condition flags and avoid use of xchg to handle rbx.
macro_rules! atomic_rmw_cas_2 {
    ($name:ident as $reexport_name:ident, $($op:tt)*) => {
        #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
        use $name as $reexport_name;
        #[cfg_attr(
            not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
            target_feature(enable = "cmpxchg16b")
        )]
        #[inline]
        unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 {
            debug_assert!(dst as usize % 16 == 0);
            debug_assert_cmpxchg16b!();
            // SAFETY: the caller must guarantee that `dst` is valid for both writes and
            // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.
            // cfg guarantees that the CPU supports CMPXCHG16B.
            //
            // See cmpxchg16b function for more.
            unsafe {
                // cmpxchg16b is always SeqCst.
                let (mut prev_lo, mut prev_hi);
                macro_rules! cmpxchg16b {
                    ($rdi:tt) => {
                        asm!(
                            // rbx is reserved by LLVM
                            "mov {rbx_tmp}, rbx",
                            // This is not single-copy atomic reads, but this is ok because subsequent
                            // CAS will check for consistency.
                            //
                            // This is based on the code generated for the first load in DW RMWs by LLVM.
                            //
                            // Note that the C++20 memory model does not allow mixed-sized atomic access,
                            // so we must use inline assembly to implement this.
                            // (i.e., byte-wise atomic based on the standard library's atomic types
                            // cannot be used here).
                            concat!("mov rax, qword ptr [", $rdi, "]"),
                            concat!("mov rdx, qword ptr [", $rdi, " + 8]"),
                            "2:",
                                $($op)*
                                concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"),
                                "jne 2b",
                            "mov rbx, {rbx_tmp}", // restore rbx
                            rbx_tmp = out(reg) _,
                            out("rcx") _,
                            out("rax") prev_lo,
                            out("rdx") prev_hi,
                            in($rdi) dst,
                            // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.
                            options(nostack),
                        )
                    };
                }
                #[cfg(target_pointer_width = "32")]
                cmpxchg16b!("edi");
                #[cfg(target_pointer_width = "64")]
                cmpxchg16b!("rdi");
                U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
            }
        }
    };
}

atomic_rmw_cas_3! {
    atomic_add_cmpxchg16b as atomic_add,
    "mov rbx, rax",
    "add rbx, rsi",
    "mov rcx, rdx",
    "adc rcx, r8",
}
atomic_rmw_cas_3! {
    atomic_sub_cmpxchg16b as atomic_sub,
    "mov rbx, rax",
    "sub rbx, rsi",
    "mov rcx, rdx",
    "sbb rcx, r8",
}
atomic_rmw_cas_3! {
    atomic_and_cmpxchg16b as atomic_and,
    "mov rbx, rax",
    "and rbx, rsi",
    "mov rcx, rdx",
    "and rcx, r8",
}
atomic_rmw_cas_3! {
    atomic_nand_cmpxchg16b as atomic_nand,
    "mov rbx, rax",
    "and rbx, rsi",
    "not rbx",
    "mov rcx, rdx",
    "and rcx, r8",
    "not rcx",
}
atomic_rmw_cas_3! {
    atomic_or_cmpxchg16b as atomic_or,
    "mov rbx, rax",
    "or rbx, rsi",
    "mov rcx, rdx",
    "or rcx, r8",
}
atomic_rmw_cas_3! {
    atomic_xor_cmpxchg16b as atomic_xor,
    "mov rbx, rax",
    "xor rbx, rsi",
    "mov rcx, rdx",
    "xor rcx, r8",
}

atomic_rmw_cas_2! {
    atomic_not_cmpxchg16b as atomic_not,
    "mov rbx, rax",
    "not rbx",
    "mov rcx, rdx",
    "not rcx",
}
atomic_rmw_cas_2! {
    atomic_neg_cmpxchg16b as atomic_neg,
    "mov rbx, rax",
    "neg rbx",
    "mov rcx, 0",
    "sbb rcx, rdx",
}

atomic_rmw_cas_3! {
    atomic_max_cmpxchg16b as atomic_max,
    "cmp rsi, rax",
    "mov rcx, r8",
    "sbb rcx, rdx",
    "mov rcx, r8",
    "cmovl rcx, rdx",
    "mov rbx, rsi",
    "cmovl rbx, rax",
}
atomic_rmw_cas_3! {
    atomic_umax_cmpxchg16b as atomic_umax,
    "cmp rsi, rax",
    "mov rcx, r8",
    "sbb rcx, rdx",
    "mov rcx, r8",
    "cmovb rcx, rdx",
    "mov rbx, rsi",
    "cmovb rbx, rax",
}
atomic_rmw_cas_3! {
    atomic_min_cmpxchg16b as atomic_min,
    "cmp rsi, rax",
    "mov rcx, r8",
    "sbb rcx, rdx",
    "mov rcx, r8",
    "cmovge rcx, rdx",
    "mov rbx, rsi",
    "cmovge rbx, rax",
}
atomic_rmw_cas_3! {
    atomic_umin_cmpxchg16b as atomic_umin,
    "cmp rsi, rax",
    "mov rcx, r8",
    "sbb rcx, rdx",
    "mov rcx, r8",
    "cmovae rcx, rdx",
    "mov rbx, rsi",
    "cmovae rbx, rax",
}

macro_rules! atomic_rmw_with_ifunc {
    (
        unsafe fn $name:ident($($arg:tt)*) $(-> $ret_ty:ty)?;
        cmpxchg16b = $cmpxchg16b_fn:ident;
        fallback = $seqcst_fallback_fn:ident;
    ) => {
        #[cfg(not(any(
            target_feature = "cmpxchg16b",
            portable_atomic_target_feature = "cmpxchg16b",
        )))]
        #[inline]
        unsafe fn $name($($arg)*, _order: Ordering) $(-> $ret_ty)? {
            fn_alias! {
                #[cfg_attr(
                    not(any(
                        target_feature = "cmpxchg16b",
                        portable_atomic_target_feature = "cmpxchg16b",
                    )),
                    target_feature(enable = "cmpxchg16b")
                )]
                unsafe fn($($arg)*) $(-> $ret_ty)?;
                // cmpxchg16b is always SeqCst.
                cmpxchg16b_seqcst_fn = $cmpxchg16b_fn(Ordering::SeqCst);
            }
            // SAFETY: the caller must uphold the safety contract.
            // we only calls cmpxchg16b_fn if cmpxchg16b is available.
            unsafe {
                ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {
                    if detect::detect().has_cmpxchg16b() {
                        cmpxchg16b_seqcst_fn
                    } else {
                        // Use SeqCst because cmpxchg16b is always SeqCst.
                        fallback::$seqcst_fallback_fn
                    }
                })
            }
        }
    };
}

atomic_rmw_with_ifunc! {
    unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_swap_cmpxchg16b;
    fallback = atomic_swap_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_add_cmpxchg16b;
    fallback = atomic_add_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_sub_cmpxchg16b;
    fallback = atomic_sub_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_and_cmpxchg16b;
    fallback = atomic_and_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_nand_cmpxchg16b;
    fallback = atomic_nand_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_or_cmpxchg16b;
    fallback = atomic_or_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_xor_cmpxchg16b;
    fallback = atomic_xor_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_max_cmpxchg16b;
    fallback = atomic_max_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_umax_cmpxchg16b;
    fallback = atomic_umax_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_min_cmpxchg16b;
    fallback = atomic_min_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128;
    cmpxchg16b = atomic_umin_cmpxchg16b;
    fallback = atomic_umin_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_not(dst: *mut u128) -> u128;
    cmpxchg16b = atomic_not_cmpxchg16b;
    fallback = atomic_not_seqcst;
}
atomic_rmw_with_ifunc! {
    unsafe fn atomic_neg(dst: *mut u128) -> u128;
    cmpxchg16b = atomic_neg_cmpxchg16b;
    fallback = atomic_neg_seqcst;
}

#[inline]
fn is_lock_free() -> bool {
    #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
    {
        // CMPXCHG16B is available at compile-time.
        true
    }
    #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
    {
        detect::detect().has_cmpxchg16b()
    }
}
const IS_ALWAYS_LOCK_FREE: bool =
    cfg!(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"));

atomic128!(AtomicI128, i128, atomic_max, atomic_min);
atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);

#[allow(clippy::undocumented_unsafe_blocks, clippy::wildcard_imports)]
#[cfg(test)]
mod tests {
    use super::*;

    test_atomic_int!(i128);
    test_atomic_int!(u128);

    // load/store/swap implementation is not affected by signedness, so it is
    // enough to test only unsigned types.
    stress_test!(u128);
}