12# define atomic_add_and_fetch_float(p, x) atomic_add_and_fetch_fl((p), (x))
13# define atomic_compare_and_swap_float(p, old_val, new_val) \
14 atomic_cas_float((p), (old_val), (new_val))
16# define atomic_fetch_and_inc_uint32(p) atomic_fetch_and_add_uint32((p), 1)
17# define atomic_fetch_and_dec_uint32(p) atomic_fetch_and_add_uint32((p), -1)
19# define CCL_LOCAL_MEM_FENCE 0
20# define ccl_barrier(flags) ((void)0)
24# ifndef __KERNEL_ONEAPI__
25# define atomic_fetch_and_add_uint32_shared atomic_fetch_and_add_uint32
28# if defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
30# define atomic_add_and_fetch_float(p, x) (atomicAdd((float *)(p), (float)(x)) + (float)(x))
32# define atomic_fetch_and_add_uint32(p, x) atomicAdd((unsigned int *)(p), (unsigned int)(x))
33# define atomic_fetch_and_sub_uint32(p, x) atomicSub((unsigned int *)(p), (unsigned int)(x))
34# define atomic_fetch_and_inc_uint32(p) atomic_fetch_and_add_uint32((p), 1)
35# define atomic_fetch_and_dec_uint32(p) atomic_fetch_and_sub_uint32((p), 1)
36# define atomic_fetch_and_or_uint32(p, x) atomicOr((unsigned int *)(p), (unsigned int)(x))
43 unsigned int int_value;
45 } new_value, prev_value, result;
46 prev_value.float_value = old_val;
47 new_value.float_value = new_val;
48 result.int_value = atomicCAS((
unsigned int *)dest, prev_value.int_value, new_value.int_value);
49 return result.float_value;
52# define atomic_store_local(p, x) (*(p) = (x))
53# define atomic_load_local(p) (*(p))
55# define CCL_LOCAL_MEM_FENCE
56# define ccl_barrier(flags) __syncthreads()
60# ifdef __KERNEL_METAL__
66# if __METAL_VERSION__ >= 300
67 return atomic_fetch_add_explicit(
68 (
ccl_global metal::atomic_float *)_source, operand, metal::memory_order_relaxed);
74 } new_value, prev_value;
75 prev_value.int_value = atomic_load_explicit(source, metal::memory_order_relaxed);
77 new_value.float_value = prev_value.float_value + operand;
78 }
while (!atomic_compare_exchange_weak_explicit(source,
79 &prev_value.int_value,
81 metal::memory_order_relaxed,
82 metal::memory_order_relaxed));
84 return new_value.float_value;
90 return atomic_fetch_add_explicit((device metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
95 return atomic_fetch_sub_explicit((device metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
100 return atomic_fetch_add_explicit((device metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
105 return atomic_fetch_sub_explicit((device metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
110 return atomic_fetch_or_explicit((device metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
116 return atomic_fetch_add_explicit(
117 (threadgroup metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
123 return atomic_fetch_sub_explicit(
124 (threadgroup metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
129 return atomic_fetch_add_explicit(
130 (threadgroup metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
135 return atomic_fetch_sub_explicit(
136 (threadgroup metal::atomic_uint *)p, 1, metal::memory_order_relaxed);
142 return atomic_fetch_or_explicit(
143 (threadgroup metal::atomic_uint *)p,
x, metal::memory_order_relaxed);
150# if __METAL_VERSION__ >= 300
151 float prev_value = old_val;
152 atomic_compare_exchange_weak_explicit((
ccl_global metal::atomic_float *)dest,
155 metal::memory_order_relaxed,
156 metal::memory_order_relaxed);
161 atomic_compare_exchange_weak_explicit((
ccl_global metal::atomic_int *)dest,
164 metal::memory_order_relaxed,
165 metal::memory_order_relaxed);
170# define atomic_store(p, x) atomic_store_explicit(p, x, metal::memory_order_relaxed)
171# define atomic_fetch(p) atomic_load_explicit(p, metal::memory_order_relaxed)
173# define atomic_store_local(p, x) \
174 atomic_store_explicit((ccl_gpu_shared metal::atomic_int *)p, x, metal::memory_order_relaxed)
175# define atomic_load_local(p) \
176 atomic_load_explicit((ccl_gpu_shared metal::atomic_int *)p, metal::memory_order_relaxed)
178# define CCL_LOCAL_MEM_FENCE metal::mem_flags::mem_threadgroup
179# define ccl_barrier(flags) threadgroup_barrier(flags)
183# ifdef __KERNEL_ONEAPI__
185# if defined(__LIBSYCL_MAJOR_VERSION) && __LIBSYCL_MAJOR_VERSION >= 9
186# define CYCLES_SYCL_GLOBAL_ADDRESS_SPACE sycl::access::address_space::global_space
188# define CYCLES_SYCL_GLOBAL_ADDRESS_SPACE \
189 sycl::access::address_space::ext_intel_global_device_space
194 sycl::atomic_ref<
float,
195 sycl::memory_order::relaxed,
196 sycl::memory_scope::device,
197 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
199 return atomic.fetch_add(
x);
206 sycl::atomic_ref<
float,
207 sycl::memory_order::relaxed,
208 sycl::memory_scope::device,
209 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
211 atomic.compare_exchange_weak(old_val, new_val);
218 sycl::atomic_ref<
unsigned int,
219 sycl::memory_order::relaxed,
220 sycl::memory_scope::device,
221 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
223 return atomic.fetch_add(
x);
228 sycl::atomic_ref<int,
229 sycl::memory_order::relaxed,
230 sycl::memory_scope::device,
231 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
233 return atomic.fetch_add(
x);
238 sycl::atomic_ref<int,
239 sycl::memory_order::relaxed,
240 sycl::memory_scope::device,
241 sycl::access::address_space::local_space>
243 return atomic.fetch_add(
x);
249 sycl::atomic_ref<
unsigned int,
250 sycl::memory_order::relaxed,
251 sycl::memory_scope::device,
252 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
254 return atomic.fetch_sub(
x);
259 sycl::atomic_ref<int,
260 sycl::memory_order::relaxed,
261 sycl::memory_scope::device,
262 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
264 return atomic.fetch_sub(
x);
279 return atomic_fetch_and_sub_uint32(p, 1);
284 return atomic_fetch_and_sub_uint32(p, 1);
290 sycl::atomic_ref<
unsigned int,
291 sycl::memory_order::relaxed,
292 sycl::memory_scope::device,
293 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
295 return atomic.fetch_or(
x);
300 sycl::atomic_ref<int,
301 sycl::memory_order::relaxed,
302 sycl::memory_scope::device,
303 CYCLES_SYCL_GLOBAL_ADDRESS_SPACE>
305 return atomic.fetch_or(
x);
310 sycl::atomic_ref<int,
311 sycl::memory_order::relaxed,
312 sycl::memory_scope::device,
313 sycl::access::address_space::local_space>
320 sycl::atomic_ref<int,
321 sycl::memory_order::relaxed,
322 sycl::memory_scope::device,
323 sycl::access::address_space::local_space>
325 return atomic.load();
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
ATOMIC_INLINE uint32_t atomic_fetch_and_or_uint32(uint32_t *p, uint32_t x)
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define ccl_device_inline
#define atomic_fetch_and_dec_uint32(p)
#define atomic_compare_and_swap_float(p, old_val, new_val)
#define atomic_fetch_and_inc_uint32(p)
#define atomic_add_and_fetch_float(p, x)
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float