Blender V5.3
parallel_active_index.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#pragma once
6
8
9/* Given an array of states, build an array of indices for which the states
10 * are active.
11 *
12 * Shared memory requirement is `sizeof(int) * (number_of_warps + 1)`. */
13
15#include "util/atomic.h"
16
17/* TODO: abstract more device differences, define `ccl_gpu_local_syncthreads`,
18 * `ccl_gpu_thread_warp`, `ccl_gpu_warp_index`, `ccl_gpu_num_warps` for all devices
19 * and keep device specific code in `compat.h`. */
20
21#ifdef __KERNEL_ONEAPI__
22
23template<typename IsActiveOp>
25 ccl_global int *ccl_restrict indices,
26 ccl_global int *ccl_restrict num_indices,
27 IsActiveOp is_active_op)
28{
29 const sycl::nd_item<1> &item_id = sycl::ext::oneapi::this_work_item::get_nd_item<1>();
30 const uint blocksize = item_id.get_local_range(0);
31
32 sycl::multi_ptr<int[GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE + 1],
33 sycl::access::address_space::local_space>
34 ptr = sycl::ext::oneapi::group_local_memory_for_overwrite<
35 int[GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE + 1]>(item_id.get_group());
36 int *warp_offset = *ptr;
37
38 /* NOTE(@nsirgien): Here we calculate the same value as below but
39 * faster for DPC++ : seems CUDA converting "%", "/", "*" based calculations below into
40 * something faster already but DPC++ doesn't, so it's better to use
41 * direct request of needed parameters - switching from this computation to computation below
42 * will cause 2.5x performance slowdown. */
43 const uint thread_index = item_id.get_local_id(0);
44 const uint thread_warp = item_id.get_sub_group().get_local_id();
45
46 const uint warp_index = item_id.get_sub_group().get_group_id();
47 const uint num_warps = item_id.get_sub_group().get_group_range()[0];
48
49 const uint state_index = item_id.get_global_id(0);
50
51 /* Test if state corresponding to this thread is active. */
52 const uint is_active = (state_index < num_states) ? is_active_op(state_index) : 0;
53#else /* !__KERNEL__ONEAPI__ */
54# ifndef __KERNEL_METAL__
55template<typename IsActiveOp>
57# endif
59 ccl_global int *indices,
60 ccl_global int *num_indices,
61# ifdef __KERNEL_METAL__
62 const uint is_active,
63 const uint blocksize,
64 const int thread_index,
65 const uint state_index,
66 const int ccl_gpu_warp_size,
67 const int thread_warp,
68 const int warp_index,
69 const int num_warps,
70 threadgroup int *warp_offset)
71{
72# else
73 IsActiveOp is_active_op)
74{
75 extern ccl_gpu_shared int warp_offset[];
76
77# ifndef __KERNEL_METAL__
78 const uint blocksize = ccl_gpu_block_dim_x;
79# endif
80
81 const uint thread_index = ccl_gpu_thread_idx_x;
82 const uint thread_warp = thread_index % ccl_gpu_warp_size;
83
84 const uint warp_index = thread_index / ccl_gpu_warp_size;
85 const uint num_warps = blocksize / ccl_gpu_warp_size;
86
87 const uint state_index = ccl_gpu_block_idx_x * blocksize + thread_index;
88
89 /* Test if state corresponding to this thread is active. */
90 const uint is_active = (state_index < num_states) ? is_active_op(state_index) : 0;
91# endif
92#endif /* !__KERNEL_ONEAPI__ */
93 /* For each thread within a warp compute how many other active states precede it. */
94#ifdef __KERNEL_ONEAPI__
95 const uint thread_offset = sycl::exclusive_scan_over_group(
96 item_id.get_sub_group(), is_active, std::plus<>());
97#else
98 const uint thread_offset = popcount(ccl_gpu_ballot(is_active) &
99 ccl_gpu_thread_mask(thread_warp));
100#endif
101
102 /* Last thread in warp stores number of active states for each warp. */
103#ifdef __KERNEL_ONEAPI__
104 if (thread_warp == item_id.get_sub_group().get_local_range()[0] - 1) {
105#else
106 if (thread_warp == ccl_gpu_warp_size - 1) {
107#endif
108 warp_offset[warp_index] = thread_offset + is_active;
109 }
110
111#ifdef __KERNEL_ONEAPI__
112 /* NOTE(@nsirgien): For us here only local memory writing (warp_offset) is important,
113 * so faster local barriers can be used. */
115#else
117#endif
118
119 /* Last thread in block converts per-warp sizes to offsets, increments global size of
120 * index array and gets offset to write to. */
121 if (thread_index == blocksize - 1) {
122 /* TODO: parallelize this. */
123 int offset = 0;
124 for (int i = 0; i < num_warps; i++) {
125 int num_active = warp_offset[i];
126 warp_offset[i] = offset;
127 offset += num_active;
128 }
129
130 const uint block_num_active = warp_offset[warp_index] + thread_offset + is_active;
131 warp_offset[num_warps] = atomic_fetch_and_add_uint32(num_indices, block_num_active);
132 }
133
134#ifdef __KERNEL_ONEAPI__
135 /* NOTE(@nsirgien): For us here only important local memory writing (warp_offset),
136 * so faster local barriers can be used. */
138#else
140#endif
141
142 /* Write to index array. */
143 if (is_active) {
144 const uint block_offset = warp_offset[num_warps];
145 indices[block_offset + warp_offset[warp_index] + thread_offset] = state_index;
146 }
147}
148
149#ifdef __KERNEL_METAL__
150
151# define gpu_parallel_active_index_array(num_states, indices, num_indices, is_active_op) \
152 const uint is_active = (ccl_gpu_global_id_x() < num_states) ? \
153 is_active_op(ccl_gpu_global_id_x()) : \
154 0; \
155 gpu_parallel_active_index_array_impl(num_states, \
156 indices, \
157 num_indices, \
158 is_active, \
159 metal_local_size, \
160 metal_local_id, \
161 metal_global_id, \
162 simdgroup_size, \
163 simd_lane_index, \
164 simd_group_index, \
165 num_simd_groups, \
166 (threadgroup int *)threadgroup_array)
167#else
168
169# define gpu_parallel_active_index_array gpu_parallel_active_index_array_impl
170
171#endif
172
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE
Definition block_sizes.h:12
#define ccl_restrict
#define ccl_global
#define ccl_gpu_block_dim_x
#define ccl_gpu_thread_idx_x
#define ccl_gpu_syncthreads()
#define ccl_gpu_warp_size
#define ccl_gpu_thread_mask(thread_warp)
#define ccl_gpu_shared
#define ccl_gpu_ballot(predicate)
#define ccl_gpu_block_idx_x
#define CCL_NAMESPACE_END
#define __KERNEL_METAL__
#define __device__
#define ccl_gpu_local_syncthreads()
const int num_states
ccl_device_inline uint popcount(const uint x)
Definition math_base.h:734
unsigned int uint
Definition BKE_text.h:55
CCL_NAMESPACE_BEGIN __device__ void gpu_parallel_active_index_array_impl(const uint num_states, ccl_global int *indices, ccl_global int *num_indices, IsActiveOp is_active_op)