24 const int volume_stack_size)
26 size_t state_size = 0;
28#define KERNEL_STRUCT_VOLUME_STACK_SIZE (volume_stack_size)
29#define KERNEL_STRUCT_BEGIN(name) for (int array_index = 0;; array_index++) {
31#ifdef __INTEGRATOR_GPU_PACKED_STATE__
32# define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
33 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
34 sizeof(gpu_state_storage<type>::gpu_type) : \
36# define KERNEL_STRUCT_MEMBER_PACKED(parent_struct, type, name, feature)
37# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) \
38 KERNEL_STRUCT_BEGIN(parent_struct) \
39 KERNEL_STRUCT_MEMBER(parent_struct, packed_##parent_struct, packed, feature)
41# define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
42 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
43 sizeof(gpu_state_storage<type>::gpu_type) : \
45# define KERNEL_STRUCT_MEMBER_PACKED KERNEL_STRUCT_MEMBER
46# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) KERNEL_STRUCT_BEGIN(parent_struct)
49#define KERNEL_STRUCT_ARRAY_MEMBER(parent_struct, type, name, feature) \
50 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
51 sizeof(gpu_state_storage<type>::gpu_type) : \
53#define KERNEL_STRUCT_END(name) \
57#define KERNEL_STRUCT_END_ARRAY(name, cpu_array_size, gpu_array_size) \
58 if (array_index >= gpu_array_size - 1) { \
67#undef KERNEL_STRUCT_VOLUME_STACK_SIZE
68#undef KERNEL_STRUCT_BEGIN
69#undef KERNEL_STRUCT_BEGIN_PACKED
70#undef KERNEL_STRUCT_MEMBER
71#undef KERNEL_STRUCT_MEMBER_PACKED
72#undef KERNEL_STRUCT_ARRAY_MEMBER
73#undef KERNEL_STRUCT_END
74#undef KERNEL_STRUCT_END_ARRAY
82 const bool *cancel_requested_flag)
83 :
PathTraceWork(device, film, device_scene, cancel_requested_flag),
84 queue_(device->gpu_queue_create()),
93 "integrator_shader_raytrace_sort_counter",
99 "integrator_shader_sort_partition_key_offsets",
154#define KERNEL_STRUCT_BEGIN(name) for (int array_index = 0;; array_index++) {
155#define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
156 integrator_state_gpu_.parent_struct.name = nullptr; \
157 if (KernelFeatureRequest(feature).test(kernel_features)) { \
158 string name_str = string_printf("%sintegrator_state_" #parent_struct "_" #name, \
159 shadow ? "shadow_" : ""); \
160 auto array = make_unique<device_only_memory<gpu_state_storage<type>::gpu_type>>( \
161 device_, name_str.c_str()); \
162 array->alloc_to_device(max_num_paths_); \
163 memcpy(&integrator_state_gpu_.parent_struct.name, \
164 &array->device_pointer, \
165 sizeof(array->device_pointer)); \
166 integrator_state_soa_.emplace_back(std::move(array)); \
168#ifdef __INTEGRATOR_GPU_PACKED_STATE__
169# define KERNEL_STRUCT_MEMBER_PACKED(parent_struct, type, name, feature) \
170 if ((KernelFeatureRequest(feature).test(kernel_features))) { \
171 string name_str = string_printf("%sintegrator_state_" #parent_struct "_" #name, \
172 shadow ? "shadow_" : ""); \
173 LOG_TRACE << "Skipping " << name_str \
174 << " -- data is packed inside integrator_state_" #parent_struct "_packed"; \
176# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) \
177 KERNEL_STRUCT_BEGIN(parent_struct) \
178 KERNEL_STRUCT_MEMBER(parent_struct, packed_##parent_struct, packed, feature)
180# define KERNEL_STRUCT_MEMBER_PACKED KERNEL_STRUCT_MEMBER
181# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) KERNEL_STRUCT_BEGIN(parent_struct)
184#define KERNEL_STRUCT_ARRAY_MEMBER(parent_struct, type, name, feature) \
185 integrator_state_gpu_.parent_struct[array_index].name = nullptr; \
186 if (KernelFeatureRequest(feature).test(kernel_features)) { \
187 string name_str = string_printf( \
188 "%sintegrator_state_" #name "_%d", shadow ? "shadow_" : "", array_index); \
189 auto array = make_unique<device_only_memory<gpu_state_storage<type>::gpu_type>>( \
190 device_, name_str.c_str()); \
191 array->alloc_to_device(max_num_paths_); \
192 memcpy(&integrator_state_gpu_.parent_struct[array_index].name, \
193 &array->device_pointer, \
194 sizeof(array->device_pointer)); \
195 integrator_state_soa_.emplace_back(std::move(array)); \
197#define KERNEL_STRUCT_END(name) \
201#define KERNEL_STRUCT_END_ARRAY(name, cpu_array_size, gpu_array_size) \
202 if (array_index >= gpu_array_size - 1) { \
206#define KERNEL_STRUCT_VOLUME_STACK_SIZE (integrator_state_soa_volume_stack_size_)
214#undef KERNEL_STRUCT_BEGIN
215#undef KERNEL_STRUCT_BEGIN_PACKED
216#undef KERNEL_STRUCT_MEMBER
217#undef KERNEL_STRUCT_MEMBER_PACKED
218#undef KERNEL_STRUCT_ARRAY_MEMBER
219#undef KERNEL_STRUCT_END
220#undef KERNEL_STRUCT_END_ARRAY
221#undef KERNEL_STRUCT_VOLUME_STACK_SIZE
224 size_t total_soa_size = 0;
226 total_soa_size += soa_memory->memory_size();
344 if (!
queue_->synchronize()) {
354 LOG_DEBUG <<
"Image cache miss in GPU kernel, updating to load requested tiles";
364 const int start_sample,
365 const int samples_num,
366 const int sample_offset)
383 int num_iterations = 0;
421 if (num_iterations) {
433 int max_num_queued = 0;
443 if (queue_counter->
num_queued[i] > max_num_queued) {
445 max_num_queued = queue_counter->
num_queued[i];
510 if (available_shadow_paths < queue_counter->num_queued[kernel] * shadow_paths_per_path) {
526 if (available_shadow_paths < shadow_paths_per_path) {
533 num_paths_limit = available_shadow_paths / shadow_paths_per_path;
539 const int num_mnee_paths =
564 const int num_queued = queue_counter->
num_queued[kernel];
644 <<
" used for path iteration, should never happen.";
650 const int num_paths_limit)
652 int d_queued_kernel = queued_kernel;
662 &
work_size, &partition_size, &num_paths_limit, &d_queued_paths, &d_queued_kernel);
675 assert(d_counter != 0 && d_prefix_sum != 0);
715 int d_queued_kernel = queued_kernel;
723 &
work_size, &d_queued_paths, &d_num_queued_paths, &d_queued_kernel);
737 const int min_compact_paths = 32;
775 const float max_overhead_factor = 2.0f;
776 const int min_compact_paths = 32;
779 num_total_paths < min_compact_paths))
797 const int max_active_path_index,
822 const int work_size = max_active_path_index;
837 if (num_compact_paths > 0) {
839 int active_states_offset = 0;
843 &d_compact_paths, &active_states_offset, &terminated_states_offset, &
work_size);
875 int num_predicted_splits = 0;
891 const int num_new_paths = num_available_paths / 2;
894 num_predicted_splits += num_scheduled_possible_split + num_new_paths;
902 while (num_paths < max_num_camera_paths) {
905 work_tiles.push_back(work_tile);
906 num_paths += work_tile.
w * work_tile.
h * work_tile.
num_samples;
914 if (work_tiles.empty() && num_paths == 0) {
921 if (work_tiles.empty()) {
939 num_predicted_splits);
946 const int num_work_tiles,
948 const int num_predicted_splits)
957 for (
int i = 0; i < num_work_tiles; i++) {
959 work_tile = work_tiles[i];
961 const int tile_work_size = work_tile.
w * work_tile.
h * work_tile.
num_samples;
966 path_index_offset += tile_work_size;
980 &d_work_tiles, &num_work_tiles, &d_render_buffer, &
max_tile_work_size, &null_ptr, &zero);
994 <<
"Invalid number of queued states for kernel "
1023 LOG_INFO <<
"Using graphics interop GPU display update.";
1026 LOG_INFO <<
"Using naive GPU display update.";
1037 const int num_samples)
1045 if (!
buffers_->buffer.device_pointer) {
1046 LOG_WARNING <<
"Request for GPU display update without allocated render buffers.";
1065 const int num_samples)
1074 const int full_x = effective_buffer_params.
full_x;
1075 const int full_y = effective_buffer_params.
full_y;
1076 const int width = effective_buffer_params.
window_width;
1078 const int final_width =
buffers_->params.window_width;
1079 const int final_height =
buffers_->params.window_height;
1081 const int texture_x = full_x - effective_big_tile_params.
full_x +
1083 const int texture_y = full_y - effective_big_tile_params.
full_y +
1113 const int num_samples)
1149 const int num_samples)
1172 if (num_active_pixels) {
1178 return num_active_pixels;
1185 num_active_pixels.
alloc(1);
1187 queue_->zero_to_device(num_active_pixels);
1194 const int reset_int =
reset;
1209 queue_->copy_from_device(num_active_pixels);
1212 return num_active_pixels.
data()[0];
1292 if (!
buffers_->buffer.device_pointer) {
1299 return queue_->synchronize();
#define GPU_PARALLEL_SORT_BLOCK_SIZE
unsigned long long int uint64_t
void image_load_requested_gpu(DeviceQueue &queue)
virtual BVHLayoutMask get_bvh_layout_mask(uint64_t kernel_features) const =0
virtual void const_copy_to(const char *name, void *host, const size_t size)=0
virtual bool should_use_graphics_interop(const GraphicsInteropDevice &, const bool=false)
device_ptr d_pixels_half_rgba
bool get_render_tile_pixels(const RenderBuffers *render_buffers, const Destination &destination) const
void graphics_interop_activate()
void copy_pixels_to_texture(const half4 *rgba_pixels, const int texture_x, const int texture_y, const int pixels_width, const int pixels_height)
GraphicsInteropBuffer & graphics_interop_get_buffer()
void graphics_interop_deactivate()
GraphicsInteropDevice graphics_interop_get_device()
bool kernel_is_shadow_path(DeviceKernel kernel)
bool copy_render_buffers_from_device() override
void compact_paths(const int num_active_paths, const int max_active_path_index, DeviceKernel terminated_paths_kernel, DeviceKernel compact_paths_kernel, DeviceKernel compact_kernel)
IntegratorStateGPU integrator_state_gpu_
device_vector< int > integrator_shader_sort_counter_
bool update_queue_counter_and_cache()
int integrator_state_soa_volume_stack_size_
void alloc_integrator_sorting()
void compact_shadow_paths(const bool force=false)
bool copy_to_display_interop(PathTraceDisplay *display, PassMode pass_mode, const int num_samples)
bool interop_use_checked_
void enqueue_adaptive_sampling_filter_y()
void enqueue_adaptive_sampling_filter_x()
void alloc_integrator_soa()
device_vector< int > num_queued_paths_
void compute_sorted_queued_paths(DeviceKernel queued_kernel, const int num_paths_limit)
int min_num_active_main_paths_
void destroy_gpu_resources(PathTraceDisplay *display) override
void alloc_work_memory() override
device_vector< int > integrator_next_main_path_index_
unique_ptr< DeviceQueue > queue_
PathTraceWorkGPU(Device *device, Film *film, DeviceScene *device_scene, const bool *cancel_requested_flag)
bool zero_render_buffers() override
bool kernel_uses_sorting(DeviceKernel kernel)
void init_execution() override
device_vector< int > integrator_shader_sort_prefix_sum_
device_vector< KernelWorkTile > work_tiles_
uint64_t integrator_state_soa_kernel_features_
void copy_to_display_naive(PathTraceDisplay *display, PassMode pass_mode, const int num_samples)
void cryptomatte_postproces() override
void alloc_integrator_path_split()
void render_samples(RenderStatistics &statistics, const int start_sample, const int samples_num, const int sample_offset) override
device_vector< IntegratorQueueCounter > integrator_queue_counter_
void denoise_volume_guiding_buffers() override
bool enqueue_work_tiles(bool &finished)
device_vector< int > queued_paths_
void compact_main_paths(const int num_active_paths)
bool has_shadow_catcher() const
bool kernel_creates_ao_paths(DeviceKernel kernel)
bool enqueue_path_iteration()
int num_active_main_paths_paths()
unique_ptr< DeviceGraphicsInterop > device_graphics_interop_
void copy_to_display(PathTraceDisplay *display, PassMode pass_mode, const int num_samples) override
device_vector< int > integrator_next_shadow_path_index_
void compute_queued_paths(DeviceKernel kernel, DeviceKernel queued_kernel)
DeviceKernel get_most_queued_kernel() const
device_vector< int > integrator_shader_raytrace_sort_counter_
bool kernel_creates_shadow_paths(DeviceKernel kernel)
device_vector< int > integrator_shader_sort_partition_key_offsets_
void alloc_integrator_queue()
void get_render_tile_film_pixels(const PassAccessor::Destination &destination, PassMode pass_mode, int num_samples)
bool should_use_graphics_interop(PathTraceDisplay *display)
vector< unique_ptr< device_memory > > integrator_state_soa_
WorkTileScheduler work_tile_scheduler_
int adaptive_sampling_convergence_check_count_active(const float threshold, bool reset)
bool copy_render_buffers_to_device() override
int max_active_main_path_index_
int adaptive_sampling_converge_filter_count_active(const float threshold, bool reset) override
int kernel_max_active_main_path_index(DeviceKernel kernel)
int shadow_catcher_count_possible_splits()
device_vector< half4 > display_rgba_half_
BufferParams effective_denoised_buffer_params_
PassAccessor::Destination get_display_destination_template(const PathTraceDisplay *display, const PassMode mode) const
unique_ptr< RenderBuffers > buffers_
PassAccessor::PassAccessInfo get_display_pass_access_info(PassMode pass_mode) const
BufferParams effective_big_tile_params_
bool has_multiple_works() const
BufferParams effective_denoised_big_tile_params_
BufferParams effective_buffer_params_
DeviceScene * device_scene_
PathTraceWork(Device *device, Film *film, DeviceScene *device_scene, const bool *cancel_requested_flag)
bool is_cancel_requested() const
void set_accelerated_rt(bool accelerated_rt)
void reset(const BufferParams &buffer_params, const int sample_start, const int samples_num, const int sample_offset, float scrambling_distance)
void set_max_num_path_states(const int max_num_path_states)
bool get_work(KernelWorkTile *work_tile, const int max_work_size=0)
device_ptr device_pointer
T * alloc(const size_t width, const size_t height=0)
@ MEM_FLAG_NO_HOST_FALLBACK
#define KERNEL_FEATURE_AO
#define KERNEL_FEATURE_NODE_RAYTRACE
#define CCL_NAMESPACE_END
const char * device_kernel_as_string(DeviceKernel kernel)
#define assert(assertion)
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int const ccl_global int const int num_active_paths
ccl_gpu_kernel_postfix const ccl_global int ccl_global float const int work_size
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int max_tile_work_size
@ DEVICE_GPU_KERNEL_INTEGRATOR_NUM
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_CHECK
@ DEVICE_KERNEL_INTEGRATOR_RESET
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SHADE_DEDICATED_LIGHT
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS
@ DEVICE_KERNEL_INTEGRATOR_SHADOW_CATCHER_COUNT_POSSIBLE_SPLITS
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_MNEE
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_SUBSURFACE
@ DEVICE_KERNEL_VOLUME_GUIDING_FILTER_X
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SHADOW
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_VOLUME_STACK
@ DEVICE_KERNEL_INTEGRATOR_SORTED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_STATES
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
@ DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_DEDICATED_LIGHT
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_STATES
@ DEVICE_KERNEL_INTEGRATOR_SHADOW_PATH_MNEE_PENDING
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_INIT_FROM_CAMERA
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_FILTER_Y
@ DEVICE_KERNEL_INTEGRATOR_SHADE_LIGHT_FORWARD
@ DEVICE_KERNEL_INTEGRATOR_SHADE_VOLUME_RAY_MARCHING
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_FILTER_X
@ DEVICE_KERNEL_INTEGRATOR_SHADE_LIGHT_NEE
@ DEVICE_KERNEL_CRYPTOMATTE_POSTPROCESS
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_INIT_FROM_BAKE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_VOLUME
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_SHADOW
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_CLOSEST
@ DEVICE_KERNEL_INTEGRATOR_SHADE_BACKGROUND
@ DEVICE_KERNEL_VOLUME_GUIDING_FILTER_Y
@ DEVICE_KERNEL_PREFIX_SUM
static CCL_NAMESPACE_BEGIN size_t estimate_single_state_size(const uint64_t kernel_features, const int volume_stack_size)
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
string string_human_readable_size(size_t size)
int num_queued[DEVICE_GPU_KERNEL_INTEGRATOR_NUM]
ccl_global int * next_shadow_path_index
ccl_global int * next_main_path_index
ccl_global int * sort_partition_key_offsets
uint sort_partition_divisor
ccl_global int * sort_key_counter[DEVICE_GPU_KERNEL_INTEGRATOR_NUM]
ccl_device_inline size_t divide_up(const size_t x, const size_t y)