Blender V5.3
atomic.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2014-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#pragma once
6
7#ifndef __KERNEL_GPU__
8
9/* Using atomic ops header from Blender. */
10# include "atomic_ops.h" // IWYU pragma: export
11
12# define atomic_add_and_fetch_float(p, x) atomic_add_and_fetch_fl((p), (x))
13# define atomic_compare_and_swap_float(p, old_val, new_val) \
14 atomic_cas_float((p), (old_val), (new_val))
15
16# define atomic_fetch_and_inc_uint32(p) atomic_fetch_and_add_uint32((p), 1)
17# define atomic_fetch_and_dec_uint32(p) atomic_fetch_and_add_uint32((p), -1)
18
19# define CCL_LOCAL_MEM_FENCE 0
20# define ccl_barrier(flags) ((void)0)
21
22#else /* __KERNEL_GPU__ */
23
24# ifndef __KERNEL_ONEAPI__
25# define atomic_fetch_and_add_uint32_shared atomic_fetch_and_add_uint32
26# endif
27
28# if defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
29
30# define atomic_add_and_fetch_float(p, x) (atomicAdd((float *)(p), (float)(x)) + (float)(x))
31
32# define atomic_fetch_and_add_uint32(p, x) atomicAdd((unsigned int *)(p), (unsigned int)(x))
33# define atomic_fetch_and_sub_uint32(p, x) atomicSub((unsigned int *)(p), (unsigned int)(x))
34# define atomic_fetch_and_inc_uint32(p) atomic_fetch_and_add_uint32((p), 1)
35# define atomic_fetch_and_dec_uint32(p) atomic_fetch_and_sub_uint32((p), 1)
36# define atomic_fetch_and_or_uint32(p, x) atomicOr((unsigned int *)(p), (unsigned int)(x))
37
38ccl_device_inline float atomic_compare_and_swap_float(volatile float *dest,
39 const float old_val,
40 const float new_val)
41{
42 union {
43 unsigned int int_value;
44 float float_value;
45 } new_value, prev_value, result;
46 prev_value.float_value = old_val;
47 new_value.float_value = new_val;
48 result.int_value = atomicCAS((unsigned int *)dest, prev_value.int_value, new_value.int_value);
49 return result.float_value;
50}
51
52# define atomic_store_local(p, x) (*(p) = (x))
53# define atomic_load_local(p) (*(p))
54
55# define CCL_LOCAL_MEM_FENCE
56# define ccl_barrier(flags) __syncthreads()
57
58# endif /* __KERNEL_CUDA__ */
59
60# ifdef __KERNEL_METAL__
61
62// global address space versions
63ccl_device_inline float atomic_add_and_fetch_float(volatile ccl_global float *_source,
64 const float operand)
65{
66# if __METAL_VERSION__ >= 300
67 return atomic_fetch_add_explicit(
68 (ccl_global metal::atomic_float *)_source, operand, metal::memory_order_relaxed);
69# else
70 volatile ccl_global metal::atomic_int *source = (ccl_global metal::atomic_int *)_source;
71 union {
72 int int_value;
73 float float_value;
74 } new_value, prev_value;
75 prev_value.int_value = atomic_load_explicit(source, metal::memory_order_relaxed);
76 do {
77 new_value.float_value = prev_value.float_value + operand;
78 } while (!atomic_compare_exchange_weak_explicit(source,
79 &prev_value.int_value,
80 new_value.int_value,
81 metal::memory_order_relaxed,
82 metal::memory_order_relaxed));
83
84 return new_value.float_value;
85# endif
86}
87
88template<class T> ccl_device_inline uint32_t atomic_fetch_and_add_uint32(device T *p, const int x)
89{
90 return atomic_fetch_add_explicit((device metal::atomic_uint *)p, x, metal::memory_order_relaxed);
91}
92
93template<class T> ccl_device_inline uint32_t atomic_fetch_and_sub_uint32(device T *p, const int x)
94{
95 return atomic_fetch_sub_explicit((device metal::atomic_uint *)p, x, metal::memory_order_relaxed);
96}
97
99{
100 return atomic_fetch_add_explicit((device metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
101}
102
103template<class T> ccl_device_inline uint32_t atomic_fetch_and_dec_uint32(device T *p)
104{
105 return atomic_fetch_sub_explicit((device metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
106}
107
108template<class T> ccl_device_inline uint32_t atomic_fetch_and_or_uint32(device T *p, const int x)
109{
110 return atomic_fetch_or_explicit((device metal::atomic_uint *)p, x, metal::memory_order_relaxed);
111}
112
113template<class T>
115{
116 return atomic_fetch_add_explicit(
117 (threadgroup metal::atomic_uint *)p, x, metal::memory_order_relaxed);
118}
119
120template<class T>
121ccl_device_inline uint32_t atomic_fetch_and_sub_uint32(threadgroup T *p, const int x)
122{
123 return atomic_fetch_sub_explicit(
124 (threadgroup metal::atomic_uint *)p, x, metal::memory_order_relaxed);
125}
126
127template<class T> ccl_device_inline uint32_t atomic_fetch_and_inc_uint32(threadgroup T *p)
128{
129 return atomic_fetch_add_explicit(
130 (threadgroup metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
131}
132
133template<class T> ccl_device_inline uint32_t atomic_fetch_and_dec_uint32(threadgroup T *p)
134{
135 return atomic_fetch_sub_explicit(
136 (threadgroup metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
137}
138
139template<class T>
141{
142 return atomic_fetch_or_explicit(
143 (threadgroup metal::atomic_uint *)p, x, metal::memory_order_relaxed);
144}
145
147 const float old_val,
148 const float new_val)
149{
150# if __METAL_VERSION__ >= 300
151 float prev_value = old_val;
152 atomic_compare_exchange_weak_explicit((ccl_global metal::atomic_float *)dest,
153 &prev_value,
154 new_val,
155 metal::memory_order_relaxed,
156 metal::memory_order_relaxed);
157 return prev_value;
158# else
159 int prev_value;
160 prev_value = __float_as_int(old_val);
161 atomic_compare_exchange_weak_explicit((ccl_global metal::atomic_int *)dest,
162 &prev_value,
163 __float_as_int(new_val),
164 metal::memory_order_relaxed,
165 metal::memory_order_relaxed);
166 return __int_as_float(prev_value);
167# endif
168}
169
170# define atomic_store(p, x) atomic_store_explicit(p, x, metal::memory_order_relaxed)
171# define atomic_fetch(p) atomic_load_explicit(p, metal::memory_order_relaxed)
172
173# define atomic_store_local(p, x) \
174 atomic_store_explicit((ccl_gpu_shared metal::atomic_int *)p, x, metal::memory_order_relaxed)
175# define atomic_load_local(p) \
176 atomic_load_explicit((ccl_gpu_shared metal::atomic_int *)p, metal::memory_order_relaxed)
177
178# define CCL_LOCAL_MEM_FENCE metal::mem_flags::mem_threadgroup
179# define ccl_barrier(flags) threadgroup_barrier(flags)
180
181# endif /* __KERNEL_METAL__ */
182
183# ifdef __KERNEL_ONEAPI__
184
185# if defined(__LIBSYCL_MAJOR_VERSION) && __LIBSYCL_MAJOR_VERSION >= 9
186# define CYCLES_SYCL_GLOBAL_ADDRESS_SPACE sycl::access::address_space::global_space
187# else
188# define CYCLES_SYCL_GLOBAL_ADDRESS_SPACE \
189 sycl::access::address_space::ext_intel_global_device_space
190# endif
191
192ccl_device_inline float atomic_add_and_fetch_float(ccl_global float *p, const float x)
193{
194 sycl::atomic_ref<float,
195 sycl::memory_order::relaxed,
196 sycl::memory_scope::device,
197 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
198 atomic(*p);
199 return atomic.fetch_add(x);
200}
201
203 float old_val,
204 float new_val)
205{
206 sycl::atomic_ref<float,
207 sycl::memory_order::relaxed,
208 sycl::memory_scope::device,
209 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
210 atomic(*source);
211 atomic.compare_exchange_weak(old_val, new_val);
212 return old_val;
213}
214
215ccl_device_inline unsigned int atomic_fetch_and_add_uint32(ccl_global unsigned int *p,
216 unsigned int x)
217{
218 sycl::atomic_ref<unsigned int,
219 sycl::memory_order::relaxed,
220 sycl::memory_scope::device,
221 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
222 atomic(*p);
223 return atomic.fetch_add(x);
224}
225
227{
228 sycl::atomic_ref<int,
229 sycl::memory_order::relaxed,
230 sycl::memory_scope::device,
231 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
232 atomic(*p);
233 return atomic.fetch_add(x);
234}
235
236ccl_device_inline int atomic_fetch_and_add_uint32_shared(int *p, const int x)
237{
238 sycl::atomic_ref<int,
239 sycl::memory_order::relaxed,
240 sycl::memory_scope::device,
241 sycl::access::address_space::local_space>
242 atomic(*p);
243 return atomic.fetch_add(x);
244}
245
246ccl_device_inline unsigned int atomic_fetch_and_sub_uint32(ccl_global unsigned int *p,
247 unsigned int x)
248{
249 sycl::atomic_ref<unsigned int,
250 sycl::memory_order::relaxed,
251 sycl::memory_scope::device,
252 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
253 atomic(*p);
254 return atomic.fetch_sub(x);
255}
256
257ccl_device_inline int atomic_fetch_and_sub_uint32(ccl_global int *p, const int x)
258{
259 sycl::atomic_ref<int,
260 sycl::memory_order::relaxed,
261 sycl::memory_scope::device,
262 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
263 atomic(*p);
264 return atomic.fetch_sub(x);
265}
266
267ccl_device_inline unsigned int atomic_fetch_and_inc_uint32(ccl_global unsigned int *p)
268{
269 return atomic_fetch_and_add_uint32(p, 1);
270}
271
273{
274 return atomic_fetch_and_add_uint32(p, 1);
275}
276
277ccl_device_inline unsigned int atomic_fetch_and_dec_uint32(ccl_global unsigned int *p)
278{
279 return atomic_fetch_and_sub_uint32(p, 1);
280}
281
283{
284 return atomic_fetch_and_sub_uint32(p, 1);
285}
286
287ccl_device_inline unsigned int atomic_fetch_and_or_uint32(ccl_global unsigned int *p,
288 unsigned int x)
289{
290 sycl::atomic_ref<unsigned int,
291 sycl::memory_order::relaxed,
292 sycl::memory_scope::device,
293 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
294 atomic(*p);
295 return atomic.fetch_or(x);
296}
297
299{
300 sycl::atomic_ref<int,
301 sycl::memory_order::relaxed,
302 sycl::memory_scope::device,
303 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
304 atomic(*p);
305 return atomic.fetch_or(x);
306}
307
308ccl_device_inline void atomic_store_local(int *p, const int x)
309{
310 sycl::atomic_ref<int,
311 sycl::memory_order::relaxed,
312 sycl::memory_scope::device,
313 sycl::access::address_space::local_space>
314 atomic(*p);
315 atomic.store(x);
316}
317
318ccl_device_inline int atomic_load_local(int *p)
319{
320 sycl::atomic_ref<int,
321 sycl::memory_order::relaxed,
322 sycl::memory_scope::device,
323 sycl::access::address_space::local_space>
324 atomic(*p);
325 return atomic.load();
326}
327
328# endif /* __KERNEL_ONEAPI__ */
329
330#endif /* __KERNEL_GPU__ */
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
ATOMIC_INLINE uint32_t atomic_fetch_and_or_uint32(uint32_t *p, uint32_t x)
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define ccl_device_inline
#define ccl_global
#define __int_as_float(x)
#define __float_as_int(x)
#define T
#define atomic_fetch_and_dec_uint32(p)
#define atomic_compare_and_swap_float(p, old_val, new_val)
#define atomic_fetch_and_inc_uint32(p)
#define atomic_add_and_fetch_float(p, x)
unsigned int uint32_t
Definition BKE_text.h:81
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float