spirv_std/arch/
atomics.rs

1#[cfg(target_arch = "spirv")]
2use core::arch::asm;
3
4use crate::{
5    float::Float,
6    integer::{Integer, SignedInteger, UnsignedInteger},
7    number::Number,
8};
9
10/// Atomically load through `ptr` using the given `SEMANTICS`. All subparts of
11/// the value that is loaded are read atomically with respect to all other
12/// atomic accesses to it within `SCOPE`.
13#[spirv_std_macros::gpu_only]
14#[doc(alias = "OpAtomicLoad")]
15#[inline]
16pub unsafe fn atomic_load<N: Number, const SCOPE: u32, const SEMANTICS: u32>(ptr: &N) -> N {
17    let mut result = N::default();
18
19    asm! {
20        "%u32 = OpTypeInt 32 0",
21        "%scope = OpConstant %u32 {scope}",
22        "%semantics = OpConstant %u32 {semantics}",
23        "%result = OpAtomicLoad _ {ptr} %scope %semantics",
24        "OpStore {result} %result",
25        scope = const SCOPE,
26        semantics = const SEMANTICS,
27        ptr = in(reg) ptr,
28        result = in(reg) &mut result
29    }
30
31    result
32}
33
34/// Atomically store through `ptr` using the given `SEMANTICS`. All subparts of
35/// `value` are written atomically with respect to all other atomic accesses to
36/// it within `SCOPE`.
37#[spirv_std_macros::gpu_only]
38#[doc(alias = "OpAtomicStore")]
39#[inline]
40pub unsafe fn atomic_store<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
41    ptr: &mut N,
42    value: N,
43) {
44    asm! {
45        "%u32 = OpTypeInt 32 0",
46        "%scope = OpConstant %u32 {scope}",
47        "%semantics = OpConstant %u32 {semantics}",
48        "%value = OpLoad _ {value}",
49        "OpAtomicStore {ptr} %scope %semantics %value",
50        scope = const SCOPE,
51        semantics = const SEMANTICS,
52        ptr = in(reg) ptr,
53        value = in(reg) &value
54    }
55}
56
57/// Perform the following steps atomically with respect to any other atomic
58/// accesses within `SCOPE` to the same location:
59///
60/// 1. Load through `ptr` to get the original value,
61/// 2. Get a new value from copying `value`, and
62/// 3. Store the new value back through `ptr`.
63///
64/// The result is the original value.
65#[spirv_std_macros::gpu_only]
66#[doc(alias = "OpAtomicExchange")]
67#[inline]
68pub unsafe fn atomic_exchange<N: Number, const SCOPE: u32, const SEMANTICS: u32>(
69    ptr: &mut N,
70    value: N,
71) -> N {
72    let mut old = N::default();
73
74    asm! {
75        "%u32 = OpTypeInt 32 0",
76        "%scope = OpConstant %u32 {scope}",
77        "%semantics = OpConstant %u32 {semantics}",
78        "%value = OpLoad _ {value}",
79        "%old = OpAtomicExchange _ {ptr} %scope %semantics %value",
80        "OpStore {old} %old",
81        scope = const SCOPE,
82        semantics = const SEMANTICS,
83        ptr = in(reg) ptr,
84        old = in(reg) &mut old,
85        value = in(reg) &value
86    }
87
88    old
89}
90
91/// Perform the following steps atomically with respect to any other atomic
92/// accesses within `SCOPE` to the same location:
93///
94/// 1. Load through `ptr` to get the original value
95/// 2. Get a new value from `value` only if the original value equals
96///    `comparator`, and
97/// 3. Store the new value back through `ptr`, only if the original value
98///    equaled `comparator`.
99///
100/// The result is the original value.
101#[spirv_std_macros::gpu_only]
102#[doc(alias = "OpAtomicCompareExchange")]
103#[inline]
104pub unsafe fn atomic_compare_exchange<
105    I: Integer,
106    const SCOPE: u32,
107    const EQUAL: u32,
108    const UNEQUAL: u32,
109>(
110    ptr: &mut I,
111    value: I,
112    comparator: I,
113) -> I {
114    let mut old = I::default();
115
116    asm! {
117        "%u32 = OpTypeInt 32 0",
118        "%scope = OpConstant %u32 {scope}",
119        "%equal = OpConstant %u32 {equal}",
120        "%unequal = OpConstant %u32 {unequal}",
121        "%value = OpLoad _ {value}",
122        "%comparator = OpLoad _ {comparator}",
123        "%old = OpAtomicCompareExchange _ {ptr} %scope %equal %unequal %value %comparator",
124        "OpStore {old} %old",
125        scope = const SCOPE,
126        equal = const EQUAL,
127        unequal = const UNEQUAL,
128        ptr = in(reg) ptr,
129        value = in(reg) &value,
130        comparator = in(reg) &comparator,
131        old = in(reg) &mut old,
132    }
133
134    old
135}
136
137/// Perform the following steps atomically with respect to any other atomic
138/// accesses within `SCOPE` to the same location:
139///
140/// 1. Load through `ptr` to get an original value,
141/// 2. Get a new value through integer addition of 1 to original value, and
142/// 3. Store the new value back through `ptr`.
143///
144/// The result is the original value.
145#[spirv_std_macros::gpu_only]
146#[doc(alias = "OpAtomicIIncrement")]
147#[inline]
148pub unsafe fn atomic_i_increment<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
149    ptr: &mut I,
150) -> I {
151    let mut old = I::default();
152
153    asm! {
154        "%u32 = OpTypeInt 32 0",
155        "%scope = OpConstant %u32 {scope}",
156        "%semantics = OpConstant %u32 {semantics}",
157        "%old = OpAtomicIIncrement _ {ptr} %scope %semantics",
158        "OpStore {old} %old",
159        scope = const SCOPE,
160        semantics = const SEMANTICS,
161        ptr = in(reg) ptr,
162        old = in(reg) &mut old
163    }
164
165    old
166}
167
168/// Perform the following steps atomically with respect to any other atomic
169/// accesses within `SCOPE` to the same location:
170///
171/// 1) load through `ptr` to get an original value,
172/// 2) get a new value through integer subtraction of 1 from original value, and
173/// 3) store the new value back through `ptr`.
174///
175/// The result is the original value.
176#[spirv_std_macros::gpu_only]
177#[doc(alias = "OpAtomicIDecrement")]
178#[inline]
179pub unsafe fn atomic_i_decrement<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
180    ptr: &mut I,
181) -> I {
182    let mut old = I::default();
183
184    asm! {
185        "%u32 = OpTypeInt 32 0",
186        "%scope = OpConstant %u32 {scope}",
187        "%semantics = OpConstant %u32 {semantics}",
188        "%old = OpAtomicIDecrement _ {ptr} %scope %semantics",
189        "OpStore {old} %old",
190        scope = const SCOPE,
191        semantics = const SEMANTICS,
192        ptr = in(reg) ptr,
193        old = in(reg) &mut old
194    }
195
196    old
197}
198
199/// Perform the following steps atomically with respect to any other atomic
200/// accesses within `SCOPE` to the same location:
201///
202/// 1) load through `ptr` to get an original value,
203/// 2) get a new value by integer addition of original value and `value`, and
204/// 3) store the new value back through `ptr`.
205///
206/// The result is the Original Value.
207#[spirv_std_macros::gpu_only]
208#[doc(alias = "OpAtomicIAdd")]
209#[inline]
210pub unsafe fn atomic_i_add<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
211    ptr: &mut I,
212    value: I,
213) -> I {
214    let mut old = I::default();
215
216    asm! {
217        "%u32 = OpTypeInt 32 0",
218        "%scope = OpConstant %u32 {scope}",
219        "%semantics = OpConstant %u32 {semantics}",
220        "%value = OpLoad _ {value}",
221        "%old = OpAtomicIAdd _ {ptr} %scope %semantics %value",
222        "OpStore {old} %old",
223        scope = const SCOPE,
224        semantics = const SEMANTICS,
225        ptr = in(reg) ptr,
226        old = in(reg) &mut old,
227        value = in(reg) &value
228    }
229
230    old
231}
232
233/// Perform the following steps atomically with respect to any other atomic
234/// accesses within `SCOPE` to the same location:
235///
236/// 1) load through `ptr` to get an original value,
237/// 2) get a new value by integer subtraction of original value and `value`, and
238/// 3) store the new value back through `ptr`.
239///
240/// The result is the Original Value.
241#[spirv_std_macros::gpu_only]
242#[doc(alias = "OpAtomicISub")]
243#[inline]
244pub unsafe fn atomic_i_sub<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
245    ptr: &mut I,
246    value: I,
247) -> I {
248    let mut old = I::default();
249
250    asm! {
251        "%u32 = OpTypeInt 32 0",
252        "%scope = OpConstant %u32 {scope}",
253        "%semantics = OpConstant %u32 {semantics}",
254        "%value = OpLoad _ {value}",
255        "%old = OpAtomicISub _ {ptr} %scope %semantics %value",
256        "OpStore {old} %old",
257        scope = const SCOPE,
258        semantics = const SEMANTICS,
259        ptr = in(reg) ptr,
260        old = in(reg) &mut old,
261        value = in(reg) &value
262    }
263
264    old
265}
266
267/// Perform the following steps atomically with respect to any other atomic
268/// accesses within Scope to the same location:
269///
270/// 1. Load through `ptr` to get an original value,
271/// 2. Get a new value by finding the smallest signed integer of original value
272///    and `value`, and
273/// 3. Store the new value back through `ptr`.
274///
275/// The result is the original value.
276#[spirv_std_macros::gpu_only]
277#[doc(alias = "OpAtomicSMin")]
278#[inline]
279pub unsafe fn atomic_s_min<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
280    ptr: &mut S,
281    value: S,
282) -> S {
283    let mut old = S::default();
284
285    asm! {
286        "%u32 = OpTypeInt 32 0",
287        "%scope = OpConstant %u32 {scope}",
288        "%semantics = OpConstant %u32 {semantics}",
289        "%value = OpLoad _ {value}",
290        "%old = OpAtomicSMin _ {ptr} %scope %semantics %value",
291        "OpStore {old} %old",
292        scope = const SCOPE,
293        semantics = const SEMANTICS,
294        ptr = in(reg) ptr,
295        old = in(reg) &mut old,
296        value = in(reg) &value
297    }
298
299    old
300}
301
302/// Perform the following steps atomically with respect to any other atomic
303/// accesses within Scope to the same location:
304///
305/// 1. Load through `ptr` to get an original value,
306/// 2. Get a new value by finding the smallest unsigned integer of original
307///    value and `value`, and
308/// 3. Store the new value back through `ptr`.
309///
310/// The result is the original value.
311#[spirv_std_macros::gpu_only]
312#[doc(alias = "OpAtomicUMin")]
313#[inline]
314pub unsafe fn atomic_u_min<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
315    ptr: &mut U,
316    value: U,
317) -> U {
318    let mut old = U::default();
319
320    asm! {
321        "%u32 = OpTypeInt 32 0",
322        "%scope = OpConstant %u32 {scope}",
323        "%semantics = OpConstant %u32 {semantics}",
324        "%value = OpLoad _ {value}",
325        "%old = OpAtomicUMin _ {ptr} %scope %semantics %value",
326        "OpStore {old} %old",
327        scope = const SCOPE,
328        semantics = const SEMANTICS,
329        ptr = in(reg) ptr,
330        old = in(reg) &mut old,
331        value = in(reg) &value
332    }
333
334    old
335}
336
337/// Perform the following steps atomically with respect to any other atomic
338/// accesses within Scope to the same location:
339///
340/// 1. Load through `ptr` to get an original value,
341/// 2. Get a new value by finding the largest signed integer of original value
342///    and `value`, and
343/// 3. Store the new value back through `ptr`.
344///
345/// The result is the original value.
346#[spirv_std_macros::gpu_only]
347#[doc(alias = "OpAtomicSMax")]
348#[inline]
349pub unsafe fn atomic_s_max<S: SignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
350    ptr: &mut S,
351    value: S,
352) -> S {
353    let mut old = S::default();
354
355    asm! {
356        "%u32 = OpTypeInt 32 0",
357        "%scope = OpConstant %u32 {scope}",
358        "%semantics = OpConstant %u32 {semantics}",
359        "%value = OpLoad _ {value}",
360        "%old = OpAtomicSMax _ {ptr} %scope %semantics %value",
361        "OpStore {old} %old",
362        scope = const SCOPE,
363        semantics = const SEMANTICS,
364        ptr = in(reg) ptr,
365        old = in(reg) &mut old,
366        value = in(reg) &value
367    }
368
369    old
370}
371
372/// Perform the following steps atomically with respect to any other atomic
373/// accesses within Scope to the same location:
374///
375/// 1. Load through `ptr` to get an original value,
376/// 2. Get a new value by finding the largest unsigned integer of original
377///    value and `value`, and
378/// 3. Store the new value back through `ptr`.
379///
380/// The result is the original value.
381#[spirv_std_macros::gpu_only]
382#[doc(alias = "OpAtomicUMax")]
383#[inline]
384pub unsafe fn atomic_u_max<U: UnsignedInteger, const SCOPE: u32, const SEMANTICS: u32>(
385    ptr: &mut U,
386    value: U,
387) -> U {
388    let mut old = U::default();
389
390    asm! {
391        "%u32 = OpTypeInt 32 0",
392        "%scope = OpConstant %u32 {scope}",
393        "%semantics = OpConstant %u32 {semantics}",
394        "%value = OpLoad _ {value}",
395        "%old = OpAtomicUMax _ {ptr} %scope %semantics %value",
396        "OpStore {old} %old",
397        scope = const SCOPE,
398        semantics = const SEMANTICS,
399        ptr = in(reg) ptr,
400        old = in(reg) &mut old,
401        value = in(reg) &value
402    }
403
404    old
405}
406
407/// Perform the following steps atomically with respect to any other atomic
408/// accesses within Scope to the same location:
409///
410/// 1. Load through `ptr` to get an original value,
411/// 2. Get a new value by the bitwise AND of the original value and `value`, and
412/// 3. Store the new value back through `ptr`.
413///
414/// The result is the original value.
415#[spirv_std_macros::gpu_only]
416#[doc(alias = "OpAtomicAnd")]
417#[inline]
418pub unsafe fn atomic_and<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
419    ptr: &mut I,
420    value: I,
421) -> I {
422    let mut old = I::default();
423
424    asm! {
425        "%u32 = OpTypeInt 32 0",
426        "%scope = OpConstant %u32 {scope}",
427        "%semantics = OpConstant %u32 {semantics}",
428        "%value = OpLoad _ {value}",
429        "%old = OpAtomicAnd _ {ptr} %scope %semantics %value",
430        "OpStore {old} %old",
431        scope = const SCOPE,
432        semantics = const SEMANTICS,
433        ptr = in(reg) ptr,
434        old = in(reg) &mut old,
435        value = in(reg) &value
436    }
437
438    old
439}
440
441/// Perform the following steps atomically with respect to any other atomic
442/// accesses within Scope to the same location:
443///
444/// 1. Load through `ptr` to get an original value,
445/// 2. Get a new value by the bitwise OR of the original value and `value`, and
446/// 3. Store the new value back through `ptr`.
447///
448/// The result is the original value.
449#[spirv_std_macros::gpu_only]
450#[doc(alias = "OpAtomicOr")]
451#[inline]
452pub unsafe fn atomic_or<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
453    ptr: &mut I,
454    value: I,
455) -> I {
456    let mut old = I::default();
457
458    asm! {
459        "%u32 = OpTypeInt 32 0",
460        "%scope = OpConstant %u32 {scope}",
461        "%semantics = OpConstant %u32 {semantics}",
462        "%value = OpLoad _ {value}",
463        "%old = OpAtomicOr _ {ptr} %scope %semantics %value",
464        "OpStore {old} %old",
465        scope = const SCOPE,
466        semantics = const SEMANTICS,
467        ptr = in(reg) ptr,
468        old = in(reg) &mut old,
469        value = in(reg) &value
470    }
471
472    old
473}
474
475/// Perform the following steps atomically with respect to any other atomic
476/// accesses within Scope to the same location:
477///
478/// 1. Load through `ptr` to get an original value,
479/// 2. Get a new value by the bitwise XOR of the original value and `value`, and
480/// 3. Store the new value back through `ptr`.
481///
482/// The result is the original value.
483#[spirv_std_macros::gpu_only]
484#[doc(alias = "OpAtomicXor")]
485#[inline]
486pub unsafe fn atomic_xor<I: Integer, const SCOPE: u32, const SEMANTICS: u32>(
487    ptr: &mut I,
488    value: I,
489) -> I {
490    let mut old = I::default();
491
492    asm! {
493        "%u32 = OpTypeInt 32 0",
494        "%scope = OpConstant %u32 {scope}",
495        "%semantics = OpConstant %u32 {semantics}",
496        "%value = OpLoad _ {value}",
497        "%old = OpAtomicXor _ {ptr} %scope %semantics %value",
498        "OpStore {old} %old",
499        scope = const SCOPE,
500        semantics = const SEMANTICS,
501        ptr = in(reg) ptr,
502        old = in(reg) &mut old,
503        value = in(reg) &value
504    }
505
506    old
507}
508
509/// Perform the following steps atomically with respect to any other atomic
510/// accesses within Scope to the same location:
511///
512/// 1. Load through `ptr` to get an original value,
513/// 2. Get a new value by finding the smallest signed integer of original value
514///    and `value`, and
515/// 3. Store the new value back through `ptr`.
516///
517/// The result is the original value.
518#[spirv_std_macros::gpu_only]
519#[doc(alias = "OpAtomicFMinEXT")]
520#[inline]
521pub unsafe fn atomic_f_min<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
522    ptr: &mut F,
523    value: F,
524) -> F {
525    let mut old = F::default();
526
527    asm! {
528        "%u32 = OpTypeInt 32 0",
529        "%scope = OpConstant %u32 {scope}",
530        "%semantics = OpConstant %u32 {semantics}",
531        "%value = OpLoad _ {value}",
532        "%old = OpAtomicFMinEXT _ {ptr} %scope %semantics %value",
533        "OpStore {old} %old",
534        scope = const SCOPE,
535        semantics = const SEMANTICS,
536        ptr = in(reg) ptr,
537        old = in(reg) &mut old,
538        value = in(reg) &value
539    }
540
541    old
542}
543
544/// Perform the following steps atomically with respect to any other atomic
545/// accesses within Scope to the same location:
546///
547/// 1. Load through `ptr` to get an original value,
548/// 2. Get a new value by finding the largest signed integer of original value
549///    and `value`, and
550/// 3. Store the new value back through `ptr`.
551///
552/// The result is the original value.
553#[spirv_std_macros::gpu_only]
554#[doc(alias = "OpAtomicFMaxEXT")]
555#[inline]
556pub unsafe fn atomic_f_max<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
557    ptr: &mut F,
558    value: F,
559) -> F {
560    let mut old = F::default();
561
562    asm! {
563        "%u32 = OpTypeInt 32 0",
564        "%scope = OpConstant %u32 {scope}",
565        "%semantics = OpConstant %u32 {semantics}",
566        "%value = OpLoad _ {value}",
567        "%old = OpAtomicFMaxEXT _ {ptr} %scope %semantics %value",
568        "OpStore {old} %old",
569        scope = const SCOPE,
570        semantics = const SEMANTICS,
571        ptr = in(reg) ptr,
572        old = in(reg) &mut old,
573        value = in(reg) &value
574    }
575
576    old
577}
578
579/// Perform the following steps atomically with respect to any other atomic
580/// accesses within `SCOPE` to the same location:
581///
582/// 1) load through `ptr` to get an original value,
583/// 2) get a new value by integer addition of original value and `value`, and
584/// 3) store the new value back through `ptr`.
585///
586/// The result is the Original Value.
587#[spirv_std_macros::gpu_only]
588#[doc(alias = "OpAtomicFAddEXT")]
589#[inline]
590pub unsafe fn atomic_f_add<F: Float, const SCOPE: u32, const SEMANTICS: u32>(
591    ptr: &mut F,
592    value: F,
593) -> F {
594    let mut old = F::default();
595
596    asm! {
597        "%u32 = OpTypeInt 32 0",
598        "%scope = OpConstant %u32 {scope}",
599        "%semantics = OpConstant %u32 {semantics}",
600        "%value = OpLoad _ {value}",
601        "%old = OpAtomicFAddEXT _ {ptr} %scope %semantics %value",
602        "OpStore {old} %old",
603        scope = const SCOPE,
604        semantics = const SEMANTICS,
605        ptr = in(reg) ptr,
606        old = in(reg) &mut old,
607        value = in(reg) &value
608    }
609
610    old
611}
spirv_std/arch/atomics.rs

spirv_std/arch/
atomics.rs
