atomic_maybe_uninit/arch/aarch64.rs
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3/*
4AArch64
5
6See "Atomic operation overview by architecture" for atomic operations in this architecture:
7https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#aarch64
8
9Refs:
10- Arm A-profile A64 Instruction Set Architecture
11 https://developer.arm.com/documentation/ddi0602/2025-06
12- C/C++ Atomics Application Binary Interface Standard for the Arm® 64-bit Architecture
13 https://github.com/ARM-software/abi-aa/blob/2025Q1/atomicsabi64/atomicsabi64.rst
14- Arm® Compiler armasm User Guide
15 https://developer.arm.com/documentation/dui0801/latest
16- Arm® Architecture Reference Manual for A-profile architecture
17 https://developer.arm.com/documentation/ddi0487/latest (PDF)
18- Arm® Architecture Reference Manual Supplement Armv8, for R-profile AArch64 architecture
19 https://developer.arm.com/documentation/ddi0600/latest (PDF)
20- portable-atomic https://github.com/taiki-e/portable-atomic
21
22See tests/asm-test/asm/atomic-maybe-uninit for generated assembly.
23*/
24
25delegate_size!(delegate_all);
26
27use core::{
28 arch::asm,
29 mem::{self, MaybeUninit},
30 sync::atomic::Ordering,
31};
32
33use crate::{
34 raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap},
35 utils::{MaybeUninit128, Pair},
36};
37
38macro_rules! atomic_rmw {
39 ($op:ident, $order:ident) => {
40 atomic_rmw!($op, $order, write = $order)
41 };
42 ($op:ident, $order:ident, write = $write:ident) => {
43 match $order {
44 Ordering::Relaxed => $op!("", "", ""),
45 Ordering::Acquire => $op!("a", "", ""),
46 Ordering::Release => $op!("", "l", ""),
47 Ordering::AcqRel => $op!("a", "l", ""),
48 // In MSVC environments, SeqCst stores/writes needs fences after writes.
49 // https://reviews.llvm.org/D141748
50 #[cfg(target_env = "msvc")]
51 Ordering::SeqCst if $write == Ordering::SeqCst => $op!("a", "l", "dmb ish"),
52 // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
53 Ordering::SeqCst => $op!("a", "l", ""),
54 _ => unreachable!(),
55 }
56 };
57}
58
59#[rustfmt::skip]
60macro_rules! atomic {
61 ($ty:ident, $suffix:tt, $val_modifier:tt, $cmp_ext:tt) => {
62 delegate_signed!(delegate_all, $ty);
63 impl AtomicLoad for $ty {
64 #[inline]
65 unsafe fn atomic_load(
66 src: *const MaybeUninit<Self>,
67 order: Ordering,
68 ) -> MaybeUninit<Self> {
69 debug_assert_atomic_unsafe_precondition!(src, $ty);
70 let out: MaybeUninit<Self>;
71
72 // SAFETY: the caller must uphold the safety contract.
73 unsafe {
74 macro_rules! atomic_load {
75 ($acquire:tt) => {
76 asm!(
77 concat!("ld", $acquire, "r", $suffix, " {out", $val_modifier, "}, [{src}]"), // atomic { out = *src }
78 src = in(reg) ptr_reg!(src),
79 out = lateout(reg) out,
80 options(nostack, preserves_flags),
81 )
82 };
83 }
84 match order {
85 Ordering::Relaxed => atomic_load!(""),
86 // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC.
87 #[cfg(target_feature = "rcpc")]
88 Ordering::Acquire => atomic_load!("ap"),
89 #[cfg(not(target_feature = "rcpc"))]
90 Ordering::Acquire => atomic_load!("a"),
91 Ordering::SeqCst => atomic_load!("a"),
92 _ => unreachable!(),
93 }
94 }
95 out
96 }
97 }
98 impl AtomicStore for $ty {
99 #[inline]
100 unsafe fn atomic_store(
101 dst: *mut MaybeUninit<Self>,
102 val: MaybeUninit<Self>,
103 order: Ordering,
104 ) {
105 debug_assert_atomic_unsafe_precondition!(dst, $ty);
106
107 // SAFETY: the caller must uphold the safety contract.
108 unsafe {
109 macro_rules! atomic_store {
110 ($release:tt, $fence:tt) => {
111 asm!(
112 concat!("st", $release, "r", $suffix, " {val", $val_modifier, "}, [{dst}]"), // atomic { *dst = val }
113 $fence, // fence
114 dst = in(reg) ptr_reg!(dst),
115 val = in(reg) val,
116 options(nostack, preserves_flags),
117 )
118 };
119 }
120 match order {
121 Ordering::Relaxed => atomic_store!("", ""),
122 Ordering::Release => atomic_store!("l", ""),
123 // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.
124 #[cfg(not(target_env = "msvc"))]
125 Ordering::SeqCst => atomic_store!("l", ""),
126 // In MSVC environments, SeqCst stores/writes needs fences after writes.
127 // https://reviews.llvm.org/D141748
128 #[cfg(target_env = "msvc")]
129 Ordering::SeqCst => atomic_store!("l", "dmb ish"),
130 _ => unreachable!(),
131 }
132 }
133 }
134 }
135 impl AtomicSwap for $ty {
136 #[inline]
137 unsafe fn atomic_swap(
138 dst: *mut MaybeUninit<Self>,
139 val: MaybeUninit<Self>,
140 order: Ordering,
141 ) -> MaybeUninit<Self> {
142 debug_assert_atomic_unsafe_precondition!(dst, $ty);
143 let mut out: MaybeUninit<Self>;
144
145 // SAFETY: the caller must uphold the safety contract.
146 unsafe {
147 #[cfg(target_feature = "lse")]
148 macro_rules! swap {
149 ($acquire:tt, $release:tt, $fence:tt) => {
150 // Refs:
151 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWP--SWPA--SWPAL--SWPL--Swap-word-or-doubleword-in-memory-
152 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWPB--SWPAB--SWPALB--SWPLB--Swap-byte-in-memory-
153 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWPH--SWPAH--SWPALH--SWPLH--Swap-halfword-in-memory-
154 asm!(
155 concat!("swp", $acquire, $release, $suffix, " {val", $val_modifier, "}, {out", $val_modifier, "}, [{dst}]"), // atomic { _x = *dst; *dst = val; out = _x }
156 $fence, // fence
157 dst = in(reg) ptr_reg!(dst),
158 val = in(reg) val,
159 out = lateout(reg) out,
160 options(nostack, preserves_flags),
161 )
162 };
163 }
164 #[cfg(not(target_feature = "lse"))]
165 macro_rules! swap {
166 ($acquire:tt, $release:tt, $fence:tt) => {
167 asm!(
168 "2:", // 'retry:
169 concat!("ld", $acquire, "xr", $suffix, " {out", $val_modifier, "}, [{dst}]"), // atomic { out = *dst; EXCLUSIVE = dst }
170 concat!("st", $release, "xr", $suffix, " {r:w}, {val", $val_modifier, "}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = val; r = 0 } else { r = 1 }; EXCLUSIVE = None }
171 "cbnz {r:w}, 2b", // if r != 0 { jump 'retry }
172 $fence, // fence
173 dst = in(reg) ptr_reg!(dst),
174 val = in(reg) val,
175 out = out(reg) out,
176 r = out(reg) _,
177 options(nostack, preserves_flags),
178 )
179 };
180 }
181 atomic_rmw!(swap, order);
182 }
183 out
184 }
185 }
186 impl AtomicCompareExchange for $ty {
187 #[inline]
188 unsafe fn atomic_compare_exchange(
189 dst: *mut MaybeUninit<Self>,
190 old: MaybeUninit<Self>,
191 new: MaybeUninit<Self>,
192 success: Ordering,
193 failure: Ordering,
194 ) -> (MaybeUninit<Self>, bool) {
195 debug_assert_atomic_unsafe_precondition!(dst, $ty);
196 let order = crate::utils::upgrade_success_ordering(success, failure);
197 let mut out: MaybeUninit<Self>;
198
199 // SAFETY: the caller must uphold the safety contract.
200 unsafe {
201 let mut r: i32;
202 #[cfg(target_feature = "lse")]
203 macro_rules! cmpxchg {
204 ($acquire:tt, $release:tt, $fence:tt) => {{
205 // Refs:
206 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CAS--CASA--CASAL--CASL--Compare-and-swap-word-or-doubleword-in-memory-
207 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASB--CASAB--CASALB--CASLB--Compare-and-swap-byte-in-memory-
208 // - https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASH--CASAH--CASALH--CASLH--Compare-and-swap-halfword-in-memory-
209 asm!(
210 // cas writes the current value to the first register,
211 // so copy the `old`'s value for later comparison.
212 concat!("mov {out", $val_modifier, "}, {old", $val_modifier, "}"), // out = old
213 concat!("cas", $acquire, $release, $suffix, " {out", $val_modifier, "}, {new", $val_modifier, "}, [{dst}]"), // atomic { if *dst == out { *dst = new } else { out = *dst } }
214 $fence, // fence
215 concat!("cmp {out", $val_modifier, "}, {old", $val_modifier, "}", $cmp_ext), // if out == old { Z = 1 } else { Z = 0 }
216 "cset {r:w}, eq", // r = Z
217 dst = in(reg) ptr_reg!(dst),
218 old = in(reg) old,
219 new = in(reg) new,
220 out = out(reg) out,
221 r = lateout(reg) r,
222 // Do not use `preserves_flags` because CMP modifies the condition flags.
223 options(nostack),
224 );
225 crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
226 (out, r != 0)
227 }};
228 }
229 #[cfg(not(target_feature = "lse"))]
230 macro_rules! cmpxchg {
231 ($acquire:tt, $release:tt, $fence:tt) => {{
232 asm!(
233 "2:", // 'retry:
234 concat!("ld", $acquire, "xr", $suffix, " {out", $val_modifier, "}, [{dst}]"), // atomic { out = *dst; EXCLUSIVE = dst }
235 concat!("cmp {out", $val_modifier, "}, {old", $val_modifier, "}", $cmp_ext), // if out == old { Z = 1 } else { Z = 0 }
236 "b.ne 3f", // if Z == 0 { jump 'cmp-fail }
237 concat!("st", $release, "xr", $suffix, " {r:w}, {new", $val_modifier, "}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = new; r = 0 } else { r = 1 }; EXCLUSIVE = None }
238 "cbnz {r:w}, 2b", // if r != 0 { jump 'retry }
239 $fence, // fence
240 "b 4f", // jump 'success
241 "3:", // 'cmp-fail:
242 "mov {r:w}, #1", // r = 1
243 "clrex", // EXCLUSIVE = None
244 "4:", // 'success:
245 dst = in(reg) ptr_reg!(dst),
246 old = in(reg) old,
247 new = in(reg) new,
248 out = out(reg) out,
249 r = out(reg) r,
250 // Do not use `preserves_flags` because CMP modifies the condition flags.
251 options(nostack),
252 );
253 crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
254 // 0 if the store was successful, 1 if no store was performed
255 (out, r == 0)
256 }};
257 }
258 atomic_rmw!(cmpxchg, order, write = success)
259 }
260 }
261 #[cfg(not(target_feature = "lse"))]
262 #[inline]
263 unsafe fn atomic_compare_exchange_weak(
264 dst: *mut MaybeUninit<Self>,
265 old: MaybeUninit<Self>,
266 new: MaybeUninit<Self>,
267 success: Ordering,
268 failure: Ordering,
269 ) -> (MaybeUninit<Self>, bool) {
270 debug_assert_atomic_unsafe_precondition!(dst, $ty);
271 let order = crate::utils::upgrade_success_ordering(success, failure);
272 let mut out: MaybeUninit<Self>;
273
274 // SAFETY: the caller must uphold the safety contract.
275 unsafe {
276 let r: i32;
277 macro_rules! cmpxchg_weak {
278 ($acquire:tt, $release:tt, $fence:tt) => {
279 asm!(
280 concat!("ld", $acquire, "xr", $suffix, " {out", $val_modifier, "}, [{dst}]"), // atomic { out = *dst; EXCLUSIVE = dst }
281 concat!("cmp {out", $val_modifier, "}, {old", $val_modifier, "}", $cmp_ext), // if out == old { Z = 1 } else { Z = 0 }
282 "b.ne 3f", // if Z == 0 { jump 'cmp-fail }
283 concat!("st", $release, "xr", $suffix, " {r:w}, {new", $val_modifier, "}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = new; r = 0 } else { r = 1 }; EXCLUSIVE = None }
284 // TODO: emit fence only when the above sc succeed?
285 // "cbnz {r:w}, 4f",
286 $fence, // fence
287 "b 4f", // jump 'success
288 "3:", // 'cmp-fail:
289 "mov {r:w}, #1", // r = 1
290 "clrex", // EXCLUSIVE = None
291 "4:", // 'success:
292 dst = in(reg) ptr_reg!(dst),
293 old = in(reg) old,
294 new = in(reg) new,
295 out = out(reg) out,
296 r = out(reg) r,
297 // Do not use `preserves_flags` because CMP modifies the condition flags.
298 options(nostack),
299 )
300 };
301 }
302 atomic_rmw!(cmpxchg_weak, order, write = success);
303 crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
304 // 0 if the store was successful, 1 if no store was performed
305 (out, r == 0)
306 }
307 }
308 }
309 };
310}
311
312atomic!(u8, "b", ":w", ", uxtb");
313atomic!(u16, "h", ":w", ", uxth");
314atomic!(u32, "", ":w", "");
315atomic!(u64, "", "", "");
316
317// There are a few ways to implement 128-bit atomic operations in AArch64.
318//
319// - LDXP/STXP loop (DW LL/SC)
320// - CASP (DWCAS) added as Armv8.1 FEAT_LSE (optional from Armv8.0, mandatory from Armv8.1)
321// - LDP/STP (DW load/store) if Armv8.4 FEAT_LSE2 (optional from Armv8.2, mandatory from Armv8.4) is available
322// - LDIAPP/STILP (DW acquire-load/release-store) added as Armv8.9 FEAT_LRCPC3 (optional from Armv8.2) (if FEAT_LSE2 is also available)
323// - LDCLRP/LDSETP/SWPP (DW RMW) added as Armv9.4 FEAT_LSE128 (optional from Armv9.3)
324//
325// If FEAT_LSE is available at compile-time, we use CASP for load/CAS. Otherwise, use LDXP/STXP loop.
326// If FEAT_LSE2 is available at compile-time, we use LDP/STP for load/store.
327// If FEAT_LSE128 is available at compile-time, we use SWPP for swap/{release,seqcst}-store.
328// If FEAT_LSE2 and FEAT_LRCPC3 are available at compile-time, we use LDIAPP/STILP for acquire-load/release-store.
329//
330// Note: FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2.
331//
332// Refs:
333// - LDXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDXP--Load-exclusive-pair-of-registers-
334// - LDAXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDAXP--Load-acquire-exclusive-pair-of-registers-
335// - STXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STXP--Store-exclusive-pair-of-registers-
336// - STLXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STLXP--Store-release-exclusive-pair-of-registers-
337//
338// Note: Load-Exclusive pair (by itself) does not guarantee atomicity; to complete an atomic
339// operation (even load/store), a corresponding Store-Exclusive pair must succeed.
340// See Arm Architecture Reference Manual for A-profile architecture
341// Section B2.2.1 "Requirements for single-copy atomicity", and
342// Section B2.9 "Synchronization and semaphores" for more.
343macro_rules! atomic128 {
344 ($ty:ident) => {
345 delegate_signed!(delegate_all, $ty);
346 impl AtomicLoad for $ty {
347 #[inline]
348 unsafe fn atomic_load(
349 src: *const MaybeUninit<Self>,
350 order: Ordering,
351 ) -> MaybeUninit<Self> {
352 debug_assert_atomic_unsafe_precondition!(src, $ty);
353 let (mut prev_lo, mut prev_hi);
354
355 #[cfg(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2"))]
356 // SAFETY: the caller must guarantee that `dst` is valid for reads,
357 // 16-byte aligned, that there are no concurrent non-atomic operations.
358 // the above cfg guarantee that the CPU supports FEAT_LSE2.
359 //
360 // Refs:
361 // - LDP https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDP--Load-pair-of-registers-
362 // - LDIAPP https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDIAPP--Load-Acquire-RCpc-ordered-pair-of-registers-
363 unsafe {
364 macro_rules! atomic_load_relaxed {
365 ($iap:tt, $dmb_ishld:tt) => {
366 asm!(
367 concat!("ld", $iap, "p {prev_lo}, {prev_hi}, [{src}]"), // atomic { prev_lo:prev_hi = *src }
368 $dmb_ishld, // fence
369 src = in(reg) ptr_reg!(src),
370 prev_hi = lateout(reg) prev_hi,
371 prev_lo = lateout(reg) prev_lo,
372 options(nostack, preserves_flags),
373 )
374 };
375 }
376 match order {
377 // if FEAT_LRCPC3 && order != relaxed => ldiapp
378 // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.
379 #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
380 Ordering::Acquire => atomic_load_relaxed!("iap", ""),
381 #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
382 Ordering::SeqCst => {
383 asm!(
384 // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.
385 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891
386 "ldar {tmp}, [{src}]", // atomic { tmp = *src }
387 "ldiapp {prev_lo}, {prev_hi}, [{src}]", // atomic { prev_lo:prev_hi = *src }
388 src = in(reg) ptr_reg!(src),
389 prev_hi = lateout(reg) prev_hi,
390 prev_lo = lateout(reg) prev_lo,
391 tmp = out(reg) _,
392 options(nostack, preserves_flags),
393 );
394 }
395
396 // else => ldp
397 Ordering::Relaxed => atomic_load_relaxed!("", ""),
398 #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
399 Ordering::Acquire => atomic_load_relaxed!("", "dmb ishld"),
400 #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
401 Ordering::SeqCst => {
402 asm!(
403 // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.
404 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891
405 "ldar {tmp}, [{src}]", // atomic { tmp = *src }
406 "ldp {prev_lo}, {prev_hi}, [{src}]", // atomic { prev_lo:prev_hi = *src }
407 "dmb ishld", // fence
408 src = in(reg) ptr_reg!(src),
409 prev_hi = lateout(reg) prev_hi,
410 prev_lo = lateout(reg) prev_lo,
411 tmp = out(reg) _,
412 options(nostack, preserves_flags),
413 );
414 }
415 _ => unreachable!(),
416 }
417 MaybeUninit128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
418 }
419 #[cfg(not(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2")))]
420 // SAFETY: the caller must uphold the safety contract.
421 unsafe {
422 #[cfg(target_feature = "lse")]
423 macro_rules! atomic_load {
424 ($acquire:tt, $release:tt) => {
425 asm!(
426 // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASP--CASPA--CASPAL--CASPL--Compare-and-swap-pair-of-words-or-doublewords-in-memory-
427 concat!("casp", $acquire, $release, " x2, x3, x2, x3, [{src}]"), // atomic { if *src == x2:x3 { *dst = x2:x3 } else { x2:x3 = *dst } }
428 src = in(reg) ptr_reg!(src),
429 // must be allocated to even/odd register pair
430 inout("x2") 0_u64 => prev_lo,
431 inout("x3") 0_u64 => prev_hi,
432 options(nostack, preserves_flags),
433 )
434 };
435 }
436 #[cfg(not(target_feature = "lse"))]
437 macro_rules! atomic_load {
438 ($acquire:tt, $release:tt) => {
439 asm!(
440 "2:", // 'retry:
441 concat!("ld", $acquire, "xp {prev_lo}, {prev_hi}, [{src}]"), // atomic { prev_lo:prev_hi = *src; EXCLUSIVE = src }
442 concat!("st", $release, "xp {r:w}, {prev_lo}, {prev_hi}, [{src}]"), // atomic { if EXCLUSIVE == src { *src = prev_lo:prev_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }
443 "cbnz {r:w}, 2b", // if r != 0 { jump 'retry }
444 src = in(reg) ptr_reg!(src),
445 prev_lo = out(reg) prev_lo,
446 prev_hi = out(reg) prev_hi,
447 r = out(reg) _,
448 options(nostack, preserves_flags),
449 )
450 };
451 }
452 match order {
453 Ordering::Relaxed => atomic_load!("", ""),
454 Ordering::Acquire => atomic_load!("a", ""),
455 Ordering::SeqCst => atomic_load!("a", "l"),
456 _ => unreachable!(),
457 }
458 MaybeUninit128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
459 }
460 }
461 }
462 impl AtomicStore for $ty {
463 #[inline]
464 unsafe fn atomic_store(
465 dst: *mut MaybeUninit<Self>,
466 val: MaybeUninit<Self>,
467 order: Ordering,
468 ) {
469 debug_assert_atomic_unsafe_precondition!(dst, $ty);
470 let val = MaybeUninit128 { whole: val };
471
472 #[cfg(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2"))]
473 // SAFETY: the caller must guarantee that `dst` is valid for writes,
474 // 16-byte aligned, that there are no concurrent non-atomic operations.
475 // the above cfg guarantee that the CPU supports FEAT_LSE2.
476 //
477 // Refs:
478 // - STP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STP--Store-pair-of-registers-
479 // - STILP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STILP--Store-release-ordered-pair-of-registers-
480 unsafe {
481 macro_rules! atomic_store {
482 ($il:tt, $acquire:tt, $release:tt) => {
483 asm!(
484 $release, // fence
485 concat!("st", $il, "p {val_lo}, {val_hi}, [{dst}]"), // atomic { *dst = val_lo:val_hi }
486 $acquire, // fence
487 dst = in(reg) ptr_reg!(dst),
488 val_lo = in(reg) val.pair.lo,
489 val_hi = in(reg) val.pair.hi,
490 options(nostack, preserves_flags),
491 )
492 };
493 }
494 // Use swpp if stp requires fences.
495 // https://reviews.llvm.org/D143506
496 #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
497 macro_rules! atomic_store_swpp {
498 ($acquire:tt, $release:tt, $fence:tt) => {
499 asm!(
500 concat!("swpp", $acquire, $release, " {val_lo}, {val_hi}, [{dst}]"), // atomic { _x = *dst; *dst = val_lo:val_hi; val_lo:val_hi = _x }
501 $fence, // fence
502 dst = in(reg) ptr_reg!(dst),
503 val_lo = inout(reg) val.pair.lo => _,
504 val_hi = inout(reg) val.pair.hi => _,
505 options(nostack, preserves_flags),
506 )
507 };
508 }
509 match order {
510 // if FEAT_LSE128 && order == seqcst => swpp
511 // Prefer swpp if stp requires fences. https://reviews.llvm.org/D143506
512 // SAFETY: cfg guarantee that the CPU supports FEAT_LSE128.
513 #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
514 Ordering::SeqCst => atomic_rmw!(atomic_store_swpp, order),
515
516 // if FEAT_LRCPC3 && order != relaxed => stilp
517 // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.
518 #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
519 Ordering::Release => atomic_store!("il", "", ""),
520 // LLVM uses store-release (dmb ish; stp); dmb ish, GCC (libatomic) and Atomics ABI Standard
521 // uses store-release (stilp) without fence for SeqCst store
522 // (https://github.com/gcc-mirror/gcc/commit/7107574958e2bed11d916a1480ef1319f15e5ffe).
523 // Considering https://reviews.llvm.org/D141748, LLVM's lowing seems
524 // to be the safer option here (I'm not convinced that the libatomic's implementation is wrong).
525 #[cfg(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3"))]
526 #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
527 Ordering::SeqCst => atomic_store!("il", "dmb ish", ""),
528
529 // if FEAT_LSE128 && order != relaxed => swpp
530 // Prefer swpp if stp requires fences. https://reviews.llvm.org/D143506
531 // SAFETY: cfg guarantee that the CPU supports FEAT_LSE128.
532 #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
533 #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
534 Ordering::Release => atomic_rmw!(atomic_store_swpp, order),
535
536 // else => stp
537 Ordering::Relaxed => atomic_store!("", "", ""),
538 #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
539 #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
540 Ordering::Release => atomic_store!("", "", "dmb ish"),
541 #[cfg(not(any(target_feature = "rcpc3", atomic_maybe_uninit_target_feature = "rcpc3")))]
542 #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
543 Ordering::SeqCst => atomic_store!("", "dmb ish", "dmb ish"),
544 _ => unreachable!(),
545 }
546 }
547 #[cfg(not(any(target_feature = "lse2", atomic_maybe_uninit_target_feature = "lse2")))]
548 // SAFETY: the caller must uphold the safety contract.
549 unsafe {
550 macro_rules! store {
551 ($acquire:tt, $release:tt, $fence:tt) => {
552 asm!(
553 "2:", // 'retry:
554 concat!("ld", $acquire, "xp xzr, {tmp}, [{dst}]"), // atomic { xzr:tmp = *dst; EXCLUSIVE = dst }
555 concat!("st", $release, "xp {tmp:w}, {val_lo}, {val_hi}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = val_lo:val_hi; tmp = 0 } else { tmp = 1 }; EXCLUSIVE = None }
556 "cbnz {tmp:w}, 2b", // if tmp != 0 { jump 'retry }
557 $fence, // fence
558 dst = in(reg) ptr_reg!(dst),
559 val_lo = in(reg) val.pair.lo,
560 val_hi = in(reg) val.pair.hi,
561 tmp = out(reg) _,
562 options(nostack, preserves_flags),
563 )
564 };
565 }
566 atomic_rmw!(store, order);
567 }
568 }
569 }
570 impl AtomicSwap for $ty {
571 #[inline]
572 unsafe fn atomic_swap(
573 dst: *mut MaybeUninit<Self>,
574 val: MaybeUninit<Self>,
575 order: Ordering,
576 ) -> MaybeUninit<Self> {
577 debug_assert_atomic_unsafe_precondition!(dst, $ty);
578 let val = MaybeUninit128 { whole: val };
579 let (mut prev_lo, mut prev_hi);
580
581 // SAFETY: the caller must uphold the safety contract.
582 unsafe {
583 #[cfg(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128"))]
584 macro_rules! swap {
585 ($acquire:tt, $release:tt, $fence:tt) => {
586 asm!(
587 concat!("swpp", $acquire, $release, " {val_lo}, {val_hi}, [{dst}]"), // atomic { _x = *dst; *dst = val_lo:val_hi; val_lo:val_hi = _x }
588 $fence, // fence
589 dst = in(reg) ptr_reg!(dst),
590 val_lo = inout(reg) val.pair.lo => prev_lo,
591 val_hi = inout(reg) val.pair.hi => prev_hi,
592 options(nostack, preserves_flags),
593 )
594 };
595 }
596 #[cfg(not(any(target_feature = "lse128", atomic_maybe_uninit_target_feature = "lse128")))]
597 macro_rules! swap {
598 ($acquire:tt, $release:tt, $fence:tt) => {
599 asm!(
600 "2:", // 'retry:
601 concat!("ld", $acquire, "xp {prev_lo}, {prev_hi}, [{dst}]"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }
602 concat!("st", $release, "xp {r:w}, {val_lo}, {val_hi}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = val_lo:val_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }
603 "cbnz {r:w}, 2b", // if r != 0 { jump 'retry }
604 $fence, // fence
605 dst = in(reg) ptr_reg!(dst),
606 val_lo = in(reg) val.pair.lo,
607 val_hi = in(reg) val.pair.hi,
608 prev_lo = out(reg) prev_lo,
609 prev_hi = out(reg) prev_hi,
610 r = out(reg) _,
611 options(nostack, preserves_flags),
612 )
613 };
614 }
615 atomic_rmw!(swap, order);
616 MaybeUninit128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
617 }
618 }
619 }
620 impl AtomicCompareExchange for $ty {
621 #[inline]
622 unsafe fn atomic_compare_exchange(
623 dst: *mut MaybeUninit<Self>,
624 old: MaybeUninit<Self>,
625 new: MaybeUninit<Self>,
626 success: Ordering,
627 failure: Ordering,
628 ) -> (MaybeUninit<Self>, bool) {
629 debug_assert_atomic_unsafe_precondition!(dst, $ty);
630 let order = crate::utils::upgrade_success_ordering(success, failure);
631 let old = MaybeUninit128 { whole: old };
632 let new = MaybeUninit128 { whole: new };
633 let (mut prev_lo, mut prev_hi);
634
635 // SAFETY: the caller must uphold the safety contract.
636 unsafe {
637 let mut r: i32;
638 #[cfg(target_feature = "lse")]
639 macro_rules! cmpxchg {
640 ($acquire:tt, $release:tt, $fence:tt) => {
641 // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASP--CASPA--CASPAL--CASPL--Compare-and-swap-pair-of-words-or-doublewords-in-memory-
642 asm!(
643 // casp writes the current value to the first register pair,
644 // so copy the `old`'s value for later comparison.
645 "mov x8, {old_lo}", // x8 = old_lo
646 "mov x9, {old_hi}", // x9 = old_hi
647 concat!("casp", $acquire, $release, " x8, x9, x4, x5, [{dst}]"), // atomic { if *src == x8:x9 { *dst = x4:x5 } else { x8:x9 = *dst } }
648 $fence, // fence
649 "cmp x8, {old_lo}", // if x8 == old_lo { Z = 1 } else { Z = 0 }
650 "ccmp x9, {old_hi}, #0, eq", // if Z == 1 { if x9 == old_hi { Z = 1 } else { Z = 0 } } else { Z = 0 }
651 "cset {r:w}, eq", // r = Z
652 dst = in(reg) ptr_reg!(dst),
653 old_lo = in(reg) old.pair.lo,
654 old_hi = in(reg) old.pair.hi,
655 r = lateout(reg) r,
656 // new pair - must be allocated to even/odd register pair
657 in("x4") new.pair.lo,
658 in("x5") new.pair.hi,
659 // prev pair - must be allocated to even/odd register pair
660 out("x8") prev_lo,
661 out("x9") prev_hi,
662 // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.
663 options(nostack),
664 )
665 };
666 }
667 #[cfg(not(target_feature = "lse"))]
668 macro_rules! cmpxchg {
669 ($acquire:tt, $release:tt, $fence:tt) => {
670 asm!(
671 "2:", // 'retry:
672 concat!("ld", $acquire, "xp {prev_lo}, {prev_hi}, [{dst}]"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }
673 "cmp {prev_lo}, {old_lo}", // if prev_lo == old_lo { Z = 1 } else { Z = 0 }
674 "ccmp {prev_hi}, {old_hi}, #0, eq", // if Z == 1 { if prev_hi == old_hi { Z = 1 } else { Z = 0 } } else { Z = 0 }
675 // write back to ensure atomicity
676 "csel {tmp_lo}, {new_lo}, {prev_lo}, eq", // if Z == 1 { tmp_lo = new_lo } else { tmp_lo = prev_lo }
677 "csel {tmp_hi}, {new_hi}, {prev_hi}, eq", // if Z == 1 { tmp_hi = new_hi } else { tmp_hi = prev_hi }
678 concat!("st", $release, "xp {r:w}, {tmp_lo}, {tmp_hi}, [{dst}]"), // atomic { if EXCLUSIVE == dst { *dst = tmp_lo:tmp_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }
679 "cbnz {r:w}, 2b", // if r != 0 { jump 'retry }
680 "cset {r:w}, eq", // r = Z
681 $fence,
682 dst = in(reg) ptr_reg!(dst),
683 old_lo = in(reg) old.pair.lo,
684 old_hi = in(reg) old.pair.hi,
685 new_lo = in(reg) new.pair.lo,
686 new_hi = in(reg) new.pair.hi,
687 prev_lo = out(reg) prev_lo,
688 prev_hi = out(reg) prev_hi,
689 r = out(reg) r,
690 tmp_lo = out(reg) _,
691 tmp_hi = out(reg) _,
692 // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.
693 options(nostack),
694 )
695 };
696 }
697 atomic_rmw!(cmpxchg, order, write = success);
698 crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test
699 (
700 MaybeUninit128 {
701 pair: Pair { lo: prev_lo, hi: prev_hi }
702 }.whole,
703 r != 0
704 )
705 }
706 }
707 }
708 };
709}
710
711atomic128!(u128);
712
713// -----------------------------------------------------------------------------
714// cfg macros
715
716#[macro_export]
717macro_rules! cfg_has_atomic_8 {
718 ($($tt:tt)*) => { $($tt)* };
719}
720#[macro_export]
721macro_rules! cfg_no_atomic_8 {
722 ($($tt:tt)*) => {};
723}
724#[macro_export]
725macro_rules! cfg_has_atomic_16 {
726 ($($tt:tt)*) => { $($tt)* };
727}
728#[macro_export]
729macro_rules! cfg_no_atomic_16 {
730 ($($tt:tt)*) => {};
731}
732#[macro_export]
733macro_rules! cfg_has_atomic_32 {
734 ($($tt:tt)*) => { $($tt)* };
735}
736#[macro_export]
737macro_rules! cfg_no_atomic_32 {
738 ($($tt:tt)*) => {};
739}
740#[macro_export]
741macro_rules! cfg_has_atomic_64 {
742 ($($tt:tt)*) => { $($tt)* };
743}
744#[macro_export]
745macro_rules! cfg_no_atomic_64 {
746 ($($tt:tt)*) => {};
747}
748#[macro_export]
749macro_rules! cfg_has_atomic_128 {
750 ($($tt:tt)*) => { $($tt)* };
751}
752#[macro_export]
753macro_rules! cfg_no_atomic_128 {
754 ($($tt:tt)*) => {};
755}
756#[macro_export]
757macro_rules! cfg_has_atomic_cas {
758 ($($tt:tt)*) => { $($tt)* };
759}
760#[macro_export]
761macro_rules! cfg_no_atomic_cas {
762 ($($tt:tt)*) => {};
763}