21#ifdef __KERNEL_ONEAPI__
23template<
typename IsActiveOp>
27 IsActiveOp is_active_op)
29 const sycl::nd_item<1> &item_id = sycl::ext::oneapi::this_work_item::get_nd_item<1>();
30 const uint blocksize = item_id.get_local_range(0);
33 sycl::access::address_space::local_space>
34 ptr = sycl::ext::oneapi::group_local_memory_for_overwrite<
36 int *warp_offset = *ptr;
43 const uint thread_index = item_id.get_local_id(0);
44 const uint thread_warp = item_id.get_sub_group().get_local_id();
46 const uint warp_index = item_id.get_sub_group().get_group_id();
47 const uint num_warps = item_id.get_sub_group().get_group_range()[0];
49 const uint state_index = item_id.get_global_id(0);
52 const uint is_active = (state_index <
num_states) ? is_active_op(state_index) : 0;
54# ifndef __KERNEL_METAL__
55template<
typename IsActiveOp>
64 const int thread_index,
65 const uint state_index,
67 const int thread_warp,
70 threadgroup
int *warp_offset)
73 IsActiveOp is_active_op)
77# ifndef __KERNEL_METAL__
90 const uint is_active = (state_index <
num_states) ? is_active_op(state_index) : 0;
94#ifdef __KERNEL_ONEAPI__
95 const uint thread_offset = sycl::exclusive_scan_over_group(
96 item_id.get_sub_group(), is_active, std::plus<>());
103#ifdef __KERNEL_ONEAPI__
104 if (thread_warp == item_id.get_sub_group().get_local_range()[0] - 1) {
108 warp_offset[warp_index] = thread_offset + is_active;
111#ifdef __KERNEL_ONEAPI__
121 if (thread_index == blocksize - 1) {
124 for (
int i = 0; i < num_warps; i++) {
125 int num_active = warp_offset[i];
126 warp_offset[i] = offset;
127 offset += num_active;
130 const uint block_num_active = warp_offset[warp_index] + thread_offset + is_active;
134#ifdef __KERNEL_ONEAPI__
144 const uint block_offset = warp_offset[num_warps];
145 indices[block_offset + warp_offset[warp_index] + thread_offset] = state_index;
149#ifdef __KERNEL_METAL__
151# define gpu_parallel_active_index_array(num_states, indices, num_indices, is_active_op) \
152 const uint is_active = (ccl_gpu_global_id_x() < num_states) ? \
153 is_active_op(ccl_gpu_global_id_x()) : \
155 gpu_parallel_active_index_array_impl(num_states, \
166 (threadgroup int *)threadgroup_array)
169# define gpu_parallel_active_index_array gpu_parallel_active_index_array_impl
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE
#define ccl_gpu_block_dim_x
#define ccl_gpu_thread_idx_x
#define ccl_gpu_syncthreads()
#define ccl_gpu_warp_size
#define ccl_gpu_thread_mask(thread_warp)
#define ccl_gpu_ballot(predicate)
#define ccl_gpu_block_idx_x
#define CCL_NAMESPACE_END
#define ccl_gpu_local_syncthreads()
ccl_device_inline uint popcount(const uint x)
CCL_NAMESPACE_BEGIN __device__ void gpu_parallel_active_index_array_impl(const uint num_states, ccl_global int *indices, ccl_global int *num_indices, IsActiveOp is_active_op)