19 const uint partition_size,
20 const uint max_shaders,
21 const uint queued_kernel,
26 const ushort local_id,
34#ifndef __KERNEL_METAL__
35# ifndef __KERNEL_ONEAPI__
45 for (uint i = local_id; i < max_shaders; i +=
local_size) {
46 atomic_store_local(&buckets[i], 0);
49#ifdef __KERNEL_ONEAPI__
59 const uint partition_start = partition_size * uint(grid_id);
60 const uint partition_end =
min(
num_states, partition_start + partition_size);
62 for (
int state_index = partition_start + uint(local_id); state_index < partition_end;
65 ushort kernel_index = d_queued_kernel[state_index];
66 if (kernel_index == queued_kernel) {
67 uint key = d_shader_sort_key[state_index] % max_shaders;
68 atomic_fetch_and_add_uint32_shared(&buckets[key], 1);
72#ifdef __KERNEL_ONEAPI__
84 for (
int i = 0; i < max_shaders; i++) {
85 partition_key_offsets[i + uint(grid_id) * (max_shaders + 1)] = offset;
86 offset = offset + atomic_load_local(&buckets[i]);
90 partition_key_offsets[max_shaders + uint(grid_id) * (max_shaders + 1)] = offset;
94#ifdef __KERNEL_METAL__
96# define gpu_parallel_sort_bucket_pass(num_states, \
102 partition_key_offsets) \
103 gpu_parallel_sort_bucket_pass_impl(num_states, \
107 (ccl_gpu_shared int *)threadgroup_array, \
113 partition_key_offsets)
115#elif defined(__KERNEL_ONEAPI__)
117# define gpu_parallel_sort_bucket_pass(num_states, \
123 partition_key_offsets) \
124 gpu_parallel_sort_bucket_pass_impl(num_states, \
128 (ccl_gpu_shared int *)threadgroup_array, \
131 partition_key_offsets)
135# define gpu_parallel_sort_bucket_pass gpu_parallel_sort_bucket_pass_impl
140 const uint partition_size,
141 const int num_states_limit,
143 const uint max_shaders,
144 const uint queued_kernel,
149 const ushort local_id,
157#ifndef __KERNEL_METAL__
158# ifndef __KERNEL_ONEAPI__
170 int partition_offset = 0;
171 for (uint i = 0; i < grid_id; i++) {
172 partition_offset += partition_key_offsets[max_shaders + i * (max_shaders + 1)];
175 ccl_global int *key_offsets = partition_key_offsets + grid_id * (max_shaders + 1);
176 for (uint i = local_id; i < max_shaders; i +=
local_size) {
177 atomic_store_local(&local_offset[i], key_offsets[i] + partition_offset);
180#ifdef __KERNEL_ONEAPI__
190 const uint partition_start = partition_size * uint(grid_id);
191 const uint partition_end =
min(
num_states, partition_start + partition_size);
193 for (
int state_index = partition_start + uint(local_id); state_index < partition_end;
196 ushort kernel_index = d_queued_kernel[state_index];
197 if (kernel_index == queued_kernel) {
198 uint key = d_shader_sort_key[state_index] % max_shaders;
199 int index = atomic_fetch_and_add_uint32_shared(&local_offset[key], 1);
200 if (
index < num_states_limit) {
201 indices[
index] = state_index;
207#ifdef __KERNEL_METAL__
209# define gpu_parallel_sort_write_pass(num_states, \
217 partition_key_offsets) \
218 gpu_parallel_sort_write_pass_impl(num_states, \
224 (ccl_gpu_shared int *)threadgroup_array, \
230 partition_key_offsets)
232#elif defined(__KERNEL_ONEAPI__)
234# define gpu_parallel_sort_write_pass(num_states, \
242 partition_key_offsets) \
243 gpu_parallel_sort_write_pass_impl(num_states, \
249 (ccl_gpu_shared int *)threadgroup_array, \
252 partition_key_offsets)
256# define gpu_parallel_sort_write_pass gpu_parallel_sort_write_pass_impl
260template<
typename GetKeyOp>
263 const int num_states_limit,
270 const int key = (state_index <
num_states) ? get_key_op(state_index) :
275 if (
index < num_states_limit) {
277 indices[
index] = state_index;
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define GPU_PARALLEL_SORTED_INDEX_INACTIVE_KEY
#define ccl_device_inline
#define ccl_gpu_block_dim_x
#define ccl_gpu_thread_idx_x
#define ccl_gpu_syncthreads()
#define ccl_gpu_block_idx_x
#define CCL_NAMESPACE_END
#define ccl_gpu_local_syncthreads()
#define __KERNEL_ONEAPI__
CCL_NAMESPACE_BEGIN ccl_device_inline void gpu_parallel_sort_bucket_pass_impl(const uint num_states, const uint partition_size, const uint max_shaders, const uint queued_kernel, ccl_global ushort *d_queued_kernel, ccl_global uint *d_shader_sort_key, ccl_global int *partition_key_offsets)
__device__ void gpu_parallel_sorted_index_array(const uint state_index, const uint num_states, const int num_states_limit, ccl_global int *indices, ccl_global int *num_indices, ccl_global int *key_counter, ccl_global int *key_prefix_sum, GetKeyOp get_key_op)
ccl_device_inline void gpu_parallel_sort_write_pass_impl(const uint num_states, const uint partition_size, const int num_states_limit, ccl_global int *indices, const uint max_shaders, const uint queued_kernel, ccl_global ushort *d_queued_kernel, ccl_global uint *d_shader_sort_key, ccl_global int *partition_key_offsets)