Blender V5.3
path_trace_work_gpu.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
6
7#include "device/device.h"
8
11
12#include "scene/scene.h"
13#include "session/buffers.h"
14
15#include "util/log.h"
16#include "util/string.h"
17
19#include "kernel/types.h"
20
22
23static size_t estimate_single_state_size(const uint64_t kernel_features,
24 const int volume_stack_size)
25{
26 size_t state_size = 0;
27
28#define KERNEL_STRUCT_VOLUME_STACK_SIZE (volume_stack_size)
29#define KERNEL_STRUCT_BEGIN(name) for (int array_index = 0;; array_index++) {
30
31#ifdef __INTEGRATOR_GPU_PACKED_STATE__
32# define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
33 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
34 sizeof(gpu_state_storage<type>::gpu_type) : \
35 0;
36# define KERNEL_STRUCT_MEMBER_PACKED(parent_struct, type, name, feature)
37# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) \
38 KERNEL_STRUCT_BEGIN(parent_struct) \
39 KERNEL_STRUCT_MEMBER(parent_struct, packed_##parent_struct, packed, feature)
40#else
41# define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
42 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
43 sizeof(gpu_state_storage<type>::gpu_type) : \
44 0;
45# define KERNEL_STRUCT_MEMBER_PACKED KERNEL_STRUCT_MEMBER
46# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) KERNEL_STRUCT_BEGIN(parent_struct)
47#endif
48
49#define KERNEL_STRUCT_ARRAY_MEMBER(parent_struct, type, name, feature) \
50 state_size += (KernelFeatureRequest(feature).test(kernel_features)) ? \
51 sizeof(gpu_state_storage<type>::gpu_type) : \
52 0;
53#define KERNEL_STRUCT_END(name) \
54 (void)array_index; \
55 break; \
56 }
57#define KERNEL_STRUCT_END_ARRAY(name, cpu_array_size, gpu_array_size) \
58 if (array_index >= gpu_array_size - 1) { \
59 break; \
60 } \
61 }
62
64
66
67#undef KERNEL_STRUCT_VOLUME_STACK_SIZE
68#undef KERNEL_STRUCT_BEGIN
69#undef KERNEL_STRUCT_BEGIN_PACKED
70#undef KERNEL_STRUCT_MEMBER
71#undef KERNEL_STRUCT_MEMBER_PACKED
72#undef KERNEL_STRUCT_ARRAY_MEMBER
73#undef KERNEL_STRUCT_END
74#undef KERNEL_STRUCT_END_ARRAY
75
76 return state_size;
77}
78
80 Film *film,
81 DeviceScene *device_scene,
82 const bool *cancel_requested_flag)
83 : PathTraceWork(device, film, device_scene, cancel_requested_flag),
84 queue_(device->gpu_queue_create()),
86 /* Use MEM_FLAG_NO_HOST_FALLBACK since having this on the GPU is more important
87 * than scene memory for performance. */
89 device, "integrator_queue_counter", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
91 device, "integrator_shader_sort_counter", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
93 "integrator_shader_raytrace_sort_counter",
97 device, "integrator_shader_sort_prefix_sum", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
99 "integrator_shader_sort_partition_key_offsets",
103 device, "integrator_next_main_path_index", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
105 device, "integrator_next_shadow_path_index", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
107 num_queued_paths_(device, "num_queued_paths", MEM_READ_WRITE, MEM_FLAG_NO_HOST_FALLBACK),
108 work_tiles_(device, "work_tiles", MEM_READ_WRITE),
109 display_rgba_half_(device, "display buffer half", MEM_READ_WRITE),
113{
114 memset(&integrator_state_gpu_, 0, sizeof(integrator_state_gpu_));
115}
116
118{
119 /* IntegrateState allocated as structure of arrays. */
120
121 /* Check if we already allocated memory for the required features.
122 * Note that both disabling and enabling features may require memory
123 * allocations, so we check for equality. */
124 const int volume_stack_size = device_scene_->data.volume_stack_size;
125 const uint64_t kernel_features = device_scene_->data.kernel_features;
126 if (integrator_state_soa_kernel_features_ == kernel_features &&
127 integrator_state_soa_volume_stack_size_ == volume_stack_size)
128 {
129 return;
130 }
132 integrator_state_soa_volume_stack_size_ = volume_stack_size;
133
134 /* Free existing state, before state size estimation which takes into account free memory. */
135 integrator_state_soa_.clear();
136
137 /* Determine the number of path states. Deferring this for as long as possible allows the
138 * back-end to make better decisions about memory availability. */
139 const size_t single_state_size = estimate_single_state_size(
141
142 max_num_paths_ = queue_->num_concurrent_states(single_state_size);
143
144 /* A 1:4 busy:total ratio was found to give good performance, independent of the
145 * total state count. This is the number of states which keeps the device occupied
146 * with work without losing performance. The renderer will add more work (when
147 * available) when the number of active paths falls below this value. */
149
150 /* Allocate a device only memory buffer before for each struct member, and then
151 * write the pointers into a struct that resides in constant memory.
152 *
153 * TODO: store float3 in separate XYZ arrays. */
154#define KERNEL_STRUCT_BEGIN(name) for (int array_index = 0;; array_index++) {
155#define KERNEL_STRUCT_MEMBER(parent_struct, type, name, feature) \
156 integrator_state_gpu_.parent_struct.name = nullptr; \
157 if (KernelFeatureRequest(feature).test(kernel_features)) { \
158 string name_str = string_printf("%sintegrator_state_" #parent_struct "_" #name, \
159 shadow ? "shadow_" : ""); \
160 auto array = make_unique<device_only_memory<gpu_state_storage<type>::gpu_type>>( \
161 device_, name_str.c_str()); \
162 array->alloc_to_device(max_num_paths_); \
163 memcpy(&integrator_state_gpu_.parent_struct.name, \
164 &array->device_pointer, \
165 sizeof(array->device_pointer)); \
166 integrator_state_soa_.emplace_back(std::move(array)); \
167 }
168#ifdef __INTEGRATOR_GPU_PACKED_STATE__
169# define KERNEL_STRUCT_MEMBER_PACKED(parent_struct, type, name, feature) \
170 if ((KernelFeatureRequest(feature).test(kernel_features))) { \
171 string name_str = string_printf("%sintegrator_state_" #parent_struct "_" #name, \
172 shadow ? "shadow_" : ""); \
173 LOG_TRACE << "Skipping " << name_str \
174 << " -- data is packed inside integrator_state_" #parent_struct "_packed"; \
175 }
176# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) \
177 KERNEL_STRUCT_BEGIN(parent_struct) \
178 KERNEL_STRUCT_MEMBER(parent_struct, packed_##parent_struct, packed, feature)
179#else
180# define KERNEL_STRUCT_MEMBER_PACKED KERNEL_STRUCT_MEMBER
181# define KERNEL_STRUCT_BEGIN_PACKED(parent_struct, feature) KERNEL_STRUCT_BEGIN(parent_struct)
182#endif
183
184#define KERNEL_STRUCT_ARRAY_MEMBER(parent_struct, type, name, feature) \
185 integrator_state_gpu_.parent_struct[array_index].name = nullptr; \
186 if (KernelFeatureRequest(feature).test(kernel_features)) { \
187 string name_str = string_printf( \
188 "%sintegrator_state_" #name "_%d", shadow ? "shadow_" : "", array_index); \
189 auto array = make_unique<device_only_memory<gpu_state_storage<type>::gpu_type>>( \
190 device_, name_str.c_str()); \
191 array->alloc_to_device(max_num_paths_); \
192 memcpy(&integrator_state_gpu_.parent_struct[array_index].name, \
193 &array->device_pointer, \
194 sizeof(array->device_pointer)); \
195 integrator_state_soa_.emplace_back(std::move(array)); \
196 }
197#define KERNEL_STRUCT_END(name) \
198 (void)array_index; \
199 break; \
200 }
201#define KERNEL_STRUCT_END_ARRAY(name, cpu_array_size, gpu_array_size) \
202 if (array_index >= gpu_array_size - 1) { \
203 break; \
204 } \
205 }
206#define KERNEL_STRUCT_VOLUME_STACK_SIZE (integrator_state_soa_volume_stack_size_)
207
208 bool shadow = false;
210
211 shadow = true;
213
214#undef KERNEL_STRUCT_BEGIN
215#undef KERNEL_STRUCT_BEGIN_PACKED
216#undef KERNEL_STRUCT_MEMBER
217#undef KERNEL_STRUCT_MEMBER_PACKED
218#undef KERNEL_STRUCT_ARRAY_MEMBER
219#undef KERNEL_STRUCT_END
220#undef KERNEL_STRUCT_END_ARRAY
221#undef KERNEL_STRUCT_VOLUME_STACK_SIZE
222
224 size_t total_soa_size = 0;
225 for (auto &&soa_memory : integrator_state_soa_) {
226 total_soa_size += soa_memory->memory_size();
227 }
228
229 LOG_TRACE << "GPU SoA state size: " << string_human_readable_size(total_soa_size);
230 }
231}
232
234{
235 if (integrator_queue_counter_.size() == 0) {
241 }
242
243 /* Allocate data for active path index arrays. */
244 if (num_queued_paths_.size() == 0) {
247 }
248
251 /* TODO: this could be skip if we had a function to just allocate on device. */
253 }
254}
255
257{
258 num_sort_partitions_ = queue_->num_sort_partitions(max_num_paths_,
259 device_scene_->data.max_shaders);
260
263
264 if (num_sort_partitions_ > 1 && queue_->supports_local_atomic_sort()) {
265 /* Allocate array for partitioned shader sorting using local atomics. */
266 const int num_offsets = (device_scene_->data.max_shaders + 1) * num_sort_partitions_;
270 }
273
276 nullptr;
277 }
278 else {
279 /* Allocate arrays for shader sorting. */
280 const int sort_buckets = device_scene_->data.max_shaders * num_sort_partitions_;
281 if (integrator_shader_sort_counter_.size() < sort_buckets) {
284
287 }
288
289 const bool use_raytrace = device_scene_->data.kernel_features & KERNEL_FEATURE_NODE_RAYTRACE;
290 if (use_raytrace && integrator_shader_raytrace_sort_counter_.size() < sort_buckets) {
293 }
294
298 (use_raytrace) ? (int *)integrator_shader_raytrace_sort_counter_.device_pointer : nullptr;
299 }
300}
301
321
329
331{
332 queue_->init_execution();
333
334 /* Copy to device side struct in constant memory. */
336 "integrator_state", &integrator_state_gpu_, sizeof(integrator_state_gpu_));
337}
338
340{
341 /* Copy stats from the device. */
342 queue_->copy_from_device(integrator_queue_counter_);
343
344 if (!queue_->synchronize()) {
345 return false;
346 }
347
348 /* Update image cache if needed. */
349 /* TODO: If the number of kernels with cache misses is small compared to the total
350 * number of queued kernels, we could try asynchronously updating the image cache
351 * while continuing to work on the majority of states? */
353 if (queue_counter->cache_miss) {
354 LOG_DEBUG << "Image cache miss in GPU kernel, updating to load requested tiles";
356 queue_counter->cache_miss = 0;
357 queue_->copy_to_device(integrator_queue_counter_);
358 }
359
360 return true;
361}
362
364 const int start_sample,
365 const int samples_num,
366 const int sample_offset)
367{
368 /* Limit number of states for the tile and rely on a greedy scheduling of tiles. This allows to
369 * add more work (because tiles are smaller, so there is higher chance that more paths will
370 * become busy after adding new tiles). This is especially important for the shadow catcher which
371 * schedules work in halves of available number of paths. */
376 start_sample,
377 samples_num,
378 sample_offset,
379 device_scene_->data.integrator.scrambling_distance);
380
382
383 int num_iterations = 0;
384 uint64_t num_busy_accum = 0;
385
386 /* TODO: set a hard limit in case of undetected kernel failures? */
387 while (true) {
388 /* Enqueue work from the scheduler, on start or when there are not enough
389 * paths to keep the device occupied. */
390 bool finished;
391 if (enqueue_work_tiles(finished)) {
393 break; /* Stop on error. */
394 }
395 }
396
397 if (is_cancel_requested()) {
398 break;
399 }
400
401 /* Stop if no more work remaining. */
402 if (finished) {
403 break;
404 }
405
406 /* Enqueue on of the path iteration kernels. */
409 break; /* Stop on error. */
410 }
411 }
412
413 if (is_cancel_requested()) {
414 break;
415 }
416
417 num_busy_accum += num_active_main_paths_paths();
418 ++num_iterations;
419 }
420
421 if (num_iterations) {
422 statistics.occupancy = float(num_busy_accum) / num_iterations / max_num_paths_;
423 }
424 else {
425 statistics.occupancy = 0.0f;
426 }
427}
428
430{
432
433 int max_num_queued = 0;
435
436 for (int i = 0; i < DEVICE_GPU_KERNEL_INTEGRATOR_NUM; i++) {
437 /* SHADOW_PATH_MNEE_PENDING is a sentinel marker on shadow slots holding an MNEE precompute
438 * payload; there is no kernel to dispatch for it. The slot transitions to a real shadow
439 * kernel (or terminates) when integrator_shade_surface runs on the main path. */
441 continue;
442 }
443 if (queue_counter->num_queued[i] > max_num_queued) {
444 kernel = (DeviceKernel)i;
445 max_num_queued = queue_counter->num_queued[i];
446 }
447 }
448
449 return kernel;
450}
451
453{
455
457 queue_->zero_to_device(integrator_queue_counter_);
460 }
461 if (device_scene_->data.kernel_features & KERNEL_FEATURE_NODE_RAYTRACE &&
463 {
465 }
466
467 /* Tiles enqueue need to know number of active paths, which is based on this counter. Zero the
468 * counter on the host side because `zero_to_device()` is not doing it. */
471 }
472
473 /* No active paths, reset to zero to avoid unnecessary work and exceeding
474 * new max number of paths if it changed. */
476}
477
479{
480 /* Find kernel to execute, with max number of queued paths. */
482
483 int num_active_paths = 0;
484 for (int i = 0; i < DEVICE_GPU_KERNEL_INTEGRATOR_NUM; i++) {
485 num_active_paths += queue_counter->num_queued[i];
486 }
487
488 if (num_active_paths == 0) {
489 return false;
490 }
491
492 /* Find kernel to execute, with max number of queued paths. */
493 const DeviceKernel kernel = get_most_queued_kernel();
494 if (kernel == DEVICE_KERNEL_NUM) {
495 return false;
496 }
497
498 /* For kernels that add shadow paths, check if there is enough space available.
499 * If not, schedule shadow kernels first to clear out the shadow paths. */
500 int num_paths_limit = INT_MAX;
501
502 if (kernel_creates_shadow_paths(kernel)) {
504
505 /* Number of shadow paths that may be created for every scheduled path. */
506 const int shadow_paths_per_path = kernel_creates_ao_paths(kernel) ? 2 : 1;
507
508 int available_shadow_paths = max_num_paths_ - integrator_next_shadow_path_index_.data()[0];
509
510 if (available_shadow_paths < queue_counter->num_queued[kernel] * shadow_paths_per_path) {
513 return true;
514 }
517 return true;
518 }
521 return true;
522 }
523
524 /* Rare corner case where there are not even enough available shadow paths
525 * to run the kernel with 1 path, in this case we must force compaction. */
526 if (available_shadow_paths < shadow_paths_per_path) {
528 available_shadow_paths = max_num_paths_ - integrator_next_shadow_path_index_.data()[0];
529 }
530 }
531
532 /* Limit number of scheduled paths to the available shadow path space. */
533 num_paths_limit = available_shadow_paths / shadow_paths_per_path;
534
536 /* MNEE should not fill in more than half the available shadow paths,
537 * so there is room to run integrator_shade_surface, which is the only
538 * kernel that can release MNEE shadow paths. */
539 const int num_mnee_paths =
541 num_paths_limit = min(num_paths_limit, max(max_num_paths_ / 2 - num_mnee_paths, 1));
542 }
543 }
544
545 /* Schedule kernel with maximum number of queued items. */
546 enqueue_path_iteration(kernel, num_paths_limit);
547
548 /* Update next shadow path index for kernels that can add shadow paths. */
549 if (kernel_creates_shadow_paths(kernel)) {
551 }
552
553 return true;
554}
555
556void PathTraceWorkGPU::enqueue_path_iteration(DeviceKernel kernel, const int num_paths_limit)
557{
558 device_ptr d_path_index = 0;
559
560 /* Create array of path indices for which this kernel is queued to be executed. */
562
564 const int num_queued = queue_counter->num_queued[kernel];
565
566 if (kernel_uses_sorting(kernel)) {
567 /* Compute array of active paths, sorted by shader. */
568 work_size = num_queued;
569 d_path_index = queued_paths_.device_pointer;
570
571 compute_sorted_queued_paths(kernel, num_paths_limit);
572 }
573 else if (num_queued < work_size) {
574 work_size = num_queued;
575 d_path_index = queued_paths_.device_pointer;
576
577 if (kernel_is_shadow_path(kernel)) {
578 /* Compute array of active shadow paths for specific kernel. */
580 }
581 else {
582 /* Compute array of active paths for specific kernel. */
584 }
585 }
586
587 work_size = min(work_size, num_paths_limit);
588
590
591 switch (kernel) {
593 const int num_path_indices = work_size;
594 device_ptr null_tiles = 0;
595 int zero = 0;
596 const DeviceKernelArguments args(&null_tiles,
597 &zero,
598 &buffers_->buffer.device_pointer,
599 &zero,
600 &d_path_index,
601 &num_path_indices);
602
603 queue_->enqueue(kernel, work_size, args);
604 break;
605 }
607 /* Closest ray intersection kernels with integrator state and render buffer. */
608 const DeviceKernelArguments args(
609 &d_path_index, &buffers_->buffer.device_pointer, &work_size);
610
611 queue_->enqueue(kernel, work_size, args);
612 break;
613 }
614
620 /* Ray intersection kernels with integrator state. */
621 const DeviceKernelArguments args(&d_path_index, &work_size);
622
623 queue_->enqueue(kernel, work_size, args);
624 break;
625 }
635 /* Shading kernels with integrator state and render buffer. */
636 const DeviceKernelArguments args(
637 &d_path_index, &buffers_->buffer.device_pointer, &work_size);
638
639 queue_->enqueue(kernel, work_size, args);
640 break;
641 }
642 default:
643 LOG_FATAL << "Unhandled kernel " << device_kernel_as_string(kernel)
644 << " used for path iteration, should never happen.";
645 break;
646 }
647}
648
650 const int num_paths_limit)
651{
652 int d_queued_kernel = queued_kernel;
653
654 /* Launch kernel to fill the active paths arrays. */
655 if (num_sort_partitions_ > 1 && queue_->supports_local_atomic_sort()) {
656 const int work_size = kernel_max_active_main_path_index(queued_kernel);
657 device_ptr d_queued_paths = queued_paths_.device_pointer;
658
659 int partition_size = (int)integrator_state_gpu_.sort_partition_divisor;
660
661 const DeviceKernelArguments args(
662 &work_size, &partition_size, &num_paths_limit, &d_queued_paths, &d_queued_kernel);
663
666 args);
669 args);
670 return;
671 }
672
673 device_ptr d_counter = (device_ptr)integrator_state_gpu_.sort_key_counter[d_queued_kernel];
675 assert(d_counter != 0 && d_prefix_sum != 0);
676
677 /* Compute prefix sum of number of active paths with each shader. */
678 {
679 const int work_size = 1;
680 int sort_buckets = device_scene_->data.max_shaders * num_sort_partitions_;
681
682 const DeviceKernelArguments args(&d_counter, &d_prefix_sum, &sort_buckets);
683
685 }
686
687 queue_->zero_to_device(num_queued_paths_);
688
689 /* Launch kernel to fill the active paths arrays. */
690 {
691 /* TODO: this could be smaller for terminated paths based on amount of work we want
692 * to schedule, and also based on num_paths_limit.
693 *
694 * Also, when the number paths is limited it may be better to prefer paths from the
695 * end of the array since compaction would need to do less work. */
696 const int work_size = kernel_max_active_main_path_index(queued_kernel);
697
698 device_ptr d_queued_paths = queued_paths_.device_pointer;
699 device_ptr d_num_queued_paths = num_queued_paths_.device_pointer;
700
702 &num_paths_limit,
703 &d_queued_paths,
704 &d_num_queued_paths,
705 &d_counter,
706 &d_prefix_sum,
707 &d_queued_kernel);
708
710 }
711}
712
714{
715 int d_queued_kernel = queued_kernel;
716
717 /* Launch kernel to fill the active paths arrays. */
718 const int work_size = kernel_max_active_main_path_index(queued_kernel);
719 device_ptr d_queued_paths = queued_paths_.device_pointer;
720 device_ptr d_num_queued_paths = num_queued_paths_.device_pointer;
721
722 const DeviceKernelArguments args(
723 &work_size, &d_queued_paths, &d_num_queued_paths, &d_queued_kernel);
724
725 queue_->zero_to_device(num_queued_paths_);
726 queue_->enqueue(kernel, work_size, args);
727}
728
730{
731 /* Early out if there is nothing that needs to be compacted. */
732 if (num_active_paths == 0) {
734 return;
735 }
736
737 const int min_compact_paths = 32;
739 max_active_main_path_index_ < min_compact_paths)
740 {
741 return;
742 }
743
744 /* Compact. */
750
751 /* Adjust max active path index now we know which part of the array is actually used. */
753}
754
756{
758 const int num_active_paths =
763
764 /* Early out if there is nothing that needs to be compacted. */
765 if (num_active_paths == 0) {
769 }
770 return;
771 }
772
773 /* Compact if we can reduce the space used by half. Not always unless forced, since
774 * compaction has a cost. */
775 const float max_overhead_factor = 2.0f;
776 const int min_compact_paths = 32;
777 const int num_total_paths = integrator_next_shadow_path_index_.data()[0];
778 if (!force && (num_total_paths < num_active_paths * max_overhead_factor ||
779 num_total_paths < min_compact_paths))
780 {
781 return;
782 }
783
784 /* Compact. */
786 num_total_paths,
790
791 /* Adjust max active path index now we know which part of the array is actually used. */
794}
795
797 const int max_active_path_index,
798 DeviceKernel terminated_paths_kernel,
799 DeviceKernel compact_paths_kernel,
800 DeviceKernel compact_kernel)
801{
802 /* Compact fragmented path states into the start of the array, moving any paths
803 * with index higher than the number of active paths into the gaps. */
804 device_ptr d_compact_paths = queued_paths_.device_pointer;
805 device_ptr d_num_queued_paths = num_queued_paths_.device_pointer;
806
807 /* Create array with terminated paths that we can write to. */
808 {
809 /* TODO: can the work size be reduced here? */
810 int offset = num_active_paths;
811 const int work_size = num_active_paths;
812
813 const DeviceKernelArguments args(&work_size, &d_compact_paths, &d_num_queued_paths, &offset);
814
815 queue_->zero_to_device(num_queued_paths_);
816 queue_->enqueue(terminated_paths_kernel, work_size, args);
817 }
818
819 /* Create array of paths that we need to compact, where the path index is bigger
820 * than the number of active paths. */
821 {
822 const int work_size = max_active_path_index;
823
824 const DeviceKernelArguments args(
825 &work_size, &d_compact_paths, &d_num_queued_paths, &num_active_paths);
826
827 queue_->zero_to_device(num_queued_paths_);
828 queue_->enqueue(compact_paths_kernel, work_size, args);
829 }
830
831 queue_->copy_from_device(num_queued_paths_);
832 queue_->synchronize();
833
834 const int num_compact_paths = num_queued_paths_.data()[0];
835
836 /* Move paths into gaps. */
837 if (num_compact_paths > 0) {
838 int work_size = num_compact_paths;
839 int active_states_offset = 0;
840 int terminated_states_offset = num_active_paths;
841
842 const DeviceKernelArguments args(
843 &d_compact_paths, &active_states_offset, &terminated_states_offset, &work_size);
844
845 queue_->enqueue(compact_kernel, work_size, args);
846 }
847}
848
850{
851 /* If there are existing paths wait them to go to intersect closest kernel, which will align the
852 * wavefront of the existing and newly added paths. */
853 /* TODO: Check whether counting new intersection kernels here will have positive affect on the
854 * performance. */
855 const DeviceKernel kernel = get_most_queued_kernel();
857 return false;
858 }
859
861
862 /* Don't schedule more work if canceling. */
863 if (is_cancel_requested()) {
864 if (num_active_paths == 0) {
865 finished = true;
866 }
867 return false;
868 }
869
870 finished = false;
871
872 vector<KernelWorkTile> work_tiles;
873
874 int max_num_camera_paths = max_num_paths_;
875 int num_predicted_splits = 0;
876
877 if (has_shadow_catcher()) {
878 /* When there are shadow catchers in the scene bounce from them will split the state. So we
879 * make sure there is enough space in the path states array to fit split states.
880 *
881 * Basically, when adding N new paths we ensure that there is 2*N available path states, so
882 * that all the new paths can be split.
883 *
884 * Note that it is possible that some of the current states can still split, so need to make
885 * sure there is enough space for them as well. */
886
887 /* Number of currently in-flight states which can still split. */
888 const int num_scheduled_possible_split = shadow_catcher_count_possible_splits();
889
890 const int num_available_paths = max_num_paths_ - num_active_paths;
891 const int num_new_paths = num_available_paths / 2;
892 max_num_camera_paths = max(num_active_paths,
893 num_active_paths + num_new_paths - num_scheduled_possible_split);
894 num_predicted_splits += num_scheduled_possible_split + num_new_paths;
895 }
896
897 /* Schedule when we're out of paths or there are too few paths to keep the
898 * device occupied. */
899 int num_paths = num_active_paths;
900 if (num_paths == 0 || num_paths < min_num_active_main_paths_) {
901 /* Get work tiles until the maximum number of path is reached. */
902 while (num_paths < max_num_camera_paths) {
903 KernelWorkTile work_tile;
904 if (work_tile_scheduler_.get_work(&work_tile, max_num_camera_paths - num_paths)) {
905 work_tiles.push_back(work_tile);
906 num_paths += work_tile.w * work_tile.h * work_tile.num_samples;
907 }
908 else {
909 break;
910 }
911 }
912
913 /* If we couldn't get any more tiles, we're done. */
914 if (work_tiles.empty() && num_paths == 0) {
915 finished = true;
916 return false;
917 }
918 }
919
920 /* Initialize paths from work tiles. */
921 if (work_tiles.empty()) {
922 return false;
923 }
924
925 /* Compact state array when number of paths becomes small relative to the
926 * known maximum path index, which makes computing active index arrays slow. */
928
929 if (has_shadow_catcher()) {
930 integrator_next_main_path_index_.data()[0] = num_paths;
932 }
933
936 work_tiles.data(),
937 work_tiles.size(),
939 num_predicted_splits);
940
941 return true;
942}
943
945 const KernelWorkTile work_tiles[],
946 const int num_work_tiles,
947 const int num_active_paths,
948 const int num_predicted_splits)
949{
950 /* Copy work tiles to device. */
951 if (work_tiles_.size() < num_work_tiles) {
952 work_tiles_.alloc(num_work_tiles);
953 }
954
955 int path_index_offset = num_active_paths;
956 int max_tile_work_size = 0;
957 for (int i = 0; i < num_work_tiles; i++) {
958 KernelWorkTile &work_tile = work_tiles_.data()[i];
959 work_tile = work_tiles[i];
960
961 const int tile_work_size = work_tile.w * work_tile.h * work_tile.num_samples;
962
963 work_tile.path_index_offset = path_index_offset;
964 work_tile.work_size = tile_work_size;
965
966 path_index_offset += tile_work_size;
967
968 max_tile_work_size = max(max_tile_work_size, tile_work_size);
969 }
970
971 queue_->copy_to_device(work_tiles_);
972
973 const device_ptr d_work_tiles = work_tiles_.device_pointer;
974 device_ptr d_render_buffer = buffers_->buffer.device_pointer;
975
976 /* Launch kernel. */
977 device_ptr null_ptr = 0;
978 int zero = 0;
979 const DeviceKernelArguments args(
980 &d_work_tiles, &num_work_tiles, &d_render_buffer, &max_tile_work_size, &null_ptr, &zero);
981
982 queue_->enqueue(kernel, max_tile_work_size * num_work_tiles, args);
983
984 max_active_main_path_index_ = path_index_offset + num_predicted_splits;
985}
986
988{
990
991 int num_paths = 0;
992 for (int i = 0; i < DEVICE_GPU_KERNEL_INTEGRATOR_NUM; i++) {
993 DCHECK_GE(queue_counter->num_queued[i], 0)
994 << "Invalid number of queued states for kernel "
995 << device_kernel_as_string(static_cast<DeviceKernel>(i));
996
998 num_paths += queue_counter->num_queued[i];
999 }
1000 }
1001
1002 return num_paths;
1003}
1004
1006{
1007 /* There are few aspects with the graphics interop when using multiple devices caused by the fact
1008 * that the PathTraceDisplay has a single texture:
1009 *
1010 * CUDA will return `CUDA_ERROR_NOT_SUPPORTED` from `cuGraphicsGLRegisterBuffer()` when
1011 * attempting to register OpenGL PBO which has been mapped. Which makes sense, because
1012 * otherwise one would run into a conflict of where the source of truth is. */
1013 if (has_multiple_works()) {
1014 return false;
1015 }
1016
1017 if (!interop_use_checked_) {
1018 Device *device = queue_->device;
1020 true);
1021
1022 if (interop_use_) {
1023 LOG_INFO << "Using graphics interop GPU display update.";
1024 }
1025 else {
1026 LOG_INFO << "Using naive GPU display update.";
1027 }
1028
1029 interop_use_checked_ = true;
1030 }
1031
1032 return interop_use_;
1033}
1034
1036 PassMode pass_mode,
1037 const int num_samples)
1038{
1039 if (device_->have_error()) {
1040 /* Don't attempt to update GPU display if the device has errors: the error state will make
1041 * wrong decisions to happen about interop, causing more chained bugs. */
1042 return;
1043 }
1044
1045 if (!buffers_->buffer.device_pointer) {
1046 LOG_WARNING << "Request for GPU display update without allocated render buffers.";
1047 return;
1048 }
1049
1050 if (should_use_graphics_interop(display)) {
1051 if (copy_to_display_interop(display, pass_mode, num_samples)) {
1052 return;
1053 }
1054
1055 /* If error happens when trying to use graphics interop fallback to the native implementation
1056 * and don't attempt to use interop for the further updates. */
1057 interop_use_ = false;
1058 }
1059
1060 copy_to_display_naive(display, pass_mode, num_samples);
1061}
1062
1064 PassMode pass_mode,
1065 const int num_samples)
1066{
1067 const BufferParams &effective_big_tile_params = (pass_mode == PassMode::DENOISED) ?
1070 const BufferParams &effective_buffer_params = (pass_mode == PassMode::DENOISED) ?
1073
1074 const int full_x = effective_buffer_params.full_x;
1075 const int full_y = effective_buffer_params.full_y;
1076 const int width = effective_buffer_params.window_width;
1077 const int height = effective_buffer_params.window_height;
1078 const int final_width = buffers_->params.window_width;
1079 const int final_height = buffers_->params.window_height;
1080
1081 const int texture_x = full_x - effective_big_tile_params.full_x +
1082 effective_buffer_params.window_x - effective_big_tile_params.window_x;
1083 const int texture_y = full_y - effective_big_tile_params.full_y +
1084 effective_buffer_params.window_y - effective_big_tile_params.window_y;
1085
1086 /* Re-allocate display memory if needed, and make sure the device pointer is allocated.
1087 *
1088 * NOTE: allocation happens to the final resolution so that no re-allocation happens on every
1089 * change of the resolution divider. However, if the display becomes smaller, shrink the
1090 * allocated memory as well. */
1091 if (display_rgba_half_.data_width != final_width ||
1092 display_rgba_half_.data_height != final_height)
1093 {
1094 display_rgba_half_.alloc(final_width, final_height);
1095 /* TODO(sergey): There should be a way to make sure device-side memory is allocated without
1096 * transferring zeroes to the device. */
1097 queue_->zero_to_device(display_rgba_half_);
1098 }
1099
1100 PassAccessor::Destination destination(film_->get_display_pass(), pass_mode);
1102
1103 get_render_tile_film_pixels(destination, pass_mode, num_samples);
1104
1105 queue_->copy_from_device(display_rgba_half_);
1106 queue_->synchronize();
1107
1108 display->copy_pixels_to_texture(display_rgba_half_.data(), texture_x, texture_y, width, height);
1109}
1110
1112 PassMode pass_mode,
1113 const int num_samples)
1114{
1116 device_graphics_interop_ = queue_->graphics_interop_create();
1117 }
1118
1119 GraphicsInteropBuffer &interop_buffer = display->graphics_interop_get_buffer();
1120 device_graphics_interop_->set_buffer(interop_buffer);
1121
1122 const device_ptr d_rgba_half = device_graphics_interop_->map();
1123 if (!d_rgba_half) {
1124 return false;
1125 }
1126
1127 PassAccessor::Destination destination = get_display_destination_template(display, pass_mode);
1128 destination.d_pixels_half_rgba = d_rgba_half;
1129
1130 get_render_tile_film_pixels(destination, pass_mode, num_samples);
1131
1132 device_graphics_interop_->unmap();
1133
1134 return true;
1135}
1136
1138{
1140 return;
1141 }
1142 display->graphics_interop_activate();
1143 device_graphics_interop_ = nullptr;
1144 display->graphics_interop_deactivate();
1145}
1146
1148 PassMode pass_mode,
1149 const int num_samples)
1150{
1151 const KernelFilm &kfilm = device_scene_->data.film;
1152
1153 const PassAccessor::PassAccessInfo pass_access_info = get_display_pass_access_info(pass_mode);
1154 if (pass_access_info.type == PASS_NONE) {
1155 return;
1156 }
1157
1158 const BufferParams &effective_buffer_params = (pass_mode == PassMode::DENOISED) ?
1161
1162 const PassAccessorGPU pass_accessor(queue_.get(), pass_access_info, kfilm.exposure, num_samples);
1163
1164 pass_accessor.get_render_tile_pixels(buffers_.get(), effective_buffer_params, destination);
1165}
1166
1168 bool reset)
1169{
1170 const int num_active_pixels = adaptive_sampling_convergence_check_count_active(threshold, reset);
1171
1172 if (num_active_pixels) {
1175 queue_->synchronize();
1176 }
1177
1178 return num_active_pixels;
1179}
1180
1182 bool reset)
1183{
1184 device_vector<uint> num_active_pixels(device_, "num_active_pixels", MEM_READ_WRITE);
1185 num_active_pixels.alloc(1);
1186
1187 queue_->zero_to_device(num_active_pixels);
1188
1190 if (!work_size) {
1191 return 0;
1192 }
1193
1194 const int reset_int = reset; /* No bool kernel arguments. */
1195
1196 const DeviceKernelArguments args(&buffers_->buffer.device_pointer,
1201 &threshold,
1202 &reset_int,
1205 &num_active_pixels.device_pointer);
1206
1208
1209 queue_->copy_from_device(num_active_pixels);
1210 queue_->synchronize();
1211
1212 return num_active_pixels.data()[0];
1213}
1214
1230
1246
1248{
1250 if (!work_size) {
1251 return;
1252 }
1253
1254 const DeviceKernelArguments args(&buffers_->buffer.device_pointer,
1255 &work_size,
1258
1260}
1261
1288
1290{
1291 /* May not exist if cancelled before rendering started. */
1292 if (!buffers_->buffer.device_pointer) {
1293 return false;
1294 }
1295
1296 queue_->copy_from_device(buffers_->buffer);
1297
1298 /* Synchronize so that the CPU-side buffer is available at the exit of this function. */
1299 return queue_->synchronize();
1300}
1301
1303{
1304 queue_->copy_to_device(buffers_->buffer);
1305
1306 /* NOTE: The direct device access to the buffers only happens within this path trace work. The
1307 * rest of communication happens via API calls which involves `copy_render_buffers_from_device()`
1308 * which will perform synchronization as needed. */
1309
1310 return true;
1311}
1312
1314{
1315 queue_->zero_to_device(buffers_->buffer);
1316
1317 return true;
1318}
1319
1321{
1322 return device_scene_->data.integrator.has_shadow_catcher;
1323}
1324
1326{
1327 if (max_active_main_path_index_ == 0) {
1328 return 0;
1329 }
1330
1331 if (!has_shadow_catcher()) {
1332 return 0;
1333 }
1334
1335 queue_->zero_to_device(num_queued_paths_);
1336
1338 device_ptr d_num_queued_paths = num_queued_paths_.device_pointer;
1339
1340 const DeviceKernelArguments args(&work_size, &d_num_queued_paths);
1341
1343 queue_->copy_from_device(num_queued_paths_);
1344 queue_->synchronize();
1345
1346 return num_queued_paths_.data()[0];
1347}
1348
1354
1364
1371
1378
1384
#define GPU_PARALLEL_SORT_BLOCK_SIZE
Definition block_sizes.h:18
unsigned long long int uint64_t
btScalar max
void reset()
int window_y
Definition buffers.h:78
int window_height
Definition buffers.h:80
int window_width
Definition buffers.h:79
NODE_DECLARE int width
Definition buffers.h:70
int window_x
Definition buffers.h:77
KernelData data
void image_load_requested_gpu(DeviceQueue &queue)
virtual BVHLayoutMask get_bvh_layout_mask(uint64_t kernel_features) const =0
virtual void const_copy_to(const char *name, void *host, const size_t size)=0
virtual bool should_use_graphics_interop(const GraphicsInteropDevice &, const bool=false)
bool have_error()
Definition film.h:29
bool get_render_tile_pixels(const RenderBuffers *render_buffers, const Destination &destination) const
void copy_pixels_to_texture(const half4 *rgba_pixels, const int texture_x, const int texture_y, const int pixels_width, const int pixels_height)
GraphicsInteropBuffer & graphics_interop_get_buffer()
GraphicsInteropDevice graphics_interop_get_device()
bool kernel_is_shadow_path(DeviceKernel kernel)
bool copy_render_buffers_from_device() override
void compact_paths(const int num_active_paths, const int max_active_path_index, DeviceKernel terminated_paths_kernel, DeviceKernel compact_paths_kernel, DeviceKernel compact_kernel)
IntegratorStateGPU integrator_state_gpu_
device_vector< int > integrator_shader_sort_counter_
void compact_shadow_paths(const bool force=false)
bool copy_to_display_interop(PathTraceDisplay *display, PassMode pass_mode, const int num_samples)
device_vector< int > num_queued_paths_
void compute_sorted_queued_paths(DeviceKernel queued_kernel, const int num_paths_limit)
void destroy_gpu_resources(PathTraceDisplay *display) override
void alloc_work_memory() override
device_vector< int > integrator_next_main_path_index_
unique_ptr< DeviceQueue > queue_
PathTraceWorkGPU(Device *device, Film *film, DeviceScene *device_scene, const bool *cancel_requested_flag)
bool zero_render_buffers() override
bool kernel_uses_sorting(DeviceKernel kernel)
void init_execution() override
device_vector< int > integrator_shader_sort_prefix_sum_
device_vector< KernelWorkTile > work_tiles_
uint64_t integrator_state_soa_kernel_features_
void copy_to_display_naive(PathTraceDisplay *display, PassMode pass_mode, const int num_samples)
void cryptomatte_postproces() override
void render_samples(RenderStatistics &statistics, const int start_sample, const int samples_num, const int sample_offset) override
device_vector< IntegratorQueueCounter > integrator_queue_counter_
void denoise_volume_guiding_buffers() override
bool enqueue_work_tiles(bool &finished)
device_vector< int > queued_paths_
void compact_main_paths(const int num_active_paths)
bool kernel_creates_ao_paths(DeviceKernel kernel)
unique_ptr< DeviceGraphicsInterop > device_graphics_interop_
void copy_to_display(PathTraceDisplay *display, PassMode pass_mode, const int num_samples) override
device_vector< int > integrator_next_shadow_path_index_
void compute_queued_paths(DeviceKernel kernel, DeviceKernel queued_kernel)
DeviceKernel get_most_queued_kernel() const
device_vector< int > integrator_shader_raytrace_sort_counter_
bool kernel_creates_shadow_paths(DeviceKernel kernel)
device_vector< int > integrator_shader_sort_partition_key_offsets_
void get_render_tile_film_pixels(const PassAccessor::Destination &destination, PassMode pass_mode, int num_samples)
bool should_use_graphics_interop(PathTraceDisplay *display)
vector< unique_ptr< device_memory > > integrator_state_soa_
WorkTileScheduler work_tile_scheduler_
int adaptive_sampling_convergence_check_count_active(const float threshold, bool reset)
bool copy_render_buffers_to_device() override
int adaptive_sampling_converge_filter_count_active(const float threshold, bool reset) override
int kernel_max_active_main_path_index(DeviceKernel kernel)
device_vector< half4 > display_rgba_half_
BufferParams effective_denoised_buffer_params_
PassAccessor::Destination get_display_destination_template(const PathTraceDisplay *display, const PassMode mode) const
unique_ptr< RenderBuffers > buffers_
PassAccessor::PassAccessInfo get_display_pass_access_info(PassMode pass_mode) const
BufferParams effective_big_tile_params_
bool has_multiple_works() const
BufferParams effective_denoised_big_tile_params_
BufferParams effective_buffer_params_
DeviceScene * device_scene_
PathTraceWork(Device *device, Film *film, DeviceScene *device_scene, const bool *cancel_requested_flag)
bool is_cancel_requested() const
void set_accelerated_rt(bool accelerated_rt)
void reset(const BufferParams &buffer_params, const int sample_start, const int samples_num, const int sample_offset, float scrambling_distance)
void set_max_num_path_states(const int max_num_path_states)
bool get_work(KernelWorkTile *work_tile, const int max_work_size=0)
size_t size() const
T * alloc(const size_t width, const size_t height=0)
@ MEM_FLAG_NO_HOST_FALLBACK
@ MEM_READ_WRITE
#define KERNEL_FEATURE_AO
#define KERNEL_FEATURE_NODE_RAYTRACE
#define CCL_NAMESPACE_END
const char * device_kernel_as_string(DeviceKernel kernel)
#define assert(assertion)
#define INT_MAX
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int const ccl_global int const int num_active_paths
ccl_gpu_kernel_postfix const ccl_global int ccl_global float const int work_size
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int max_tile_work_size
@ DEVICE_GPU_KERNEL_INTEGRATOR_NUM
@ BVH_LAYOUT_OPTIX
@ PASS_NONE
DeviceKernel
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_CHECK
@ DEVICE_KERNEL_INTEGRATOR_RESET
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SHADE_DEDICATED_LIGHT
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS
@ DEVICE_KERNEL_INTEGRATOR_SHADOW_CATCHER_COUNT_POSSIBLE_SPLITS
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_MNEE
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_SUBSURFACE
@ DEVICE_KERNEL_VOLUME_GUIDING_FILTER_X
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SHADOW
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_VOLUME_STACK
@ DEVICE_KERNEL_INTEGRATOR_SORTED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_STATES
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
@ DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_DEDICATED_LIGHT
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_STATES
@ DEVICE_KERNEL_INTEGRATOR_SHADOW_PATH_MNEE_PENDING
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_NUM
@ DEVICE_KERNEL_INTEGRATOR_INIT_FROM_CAMERA
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_FILTER_Y
@ DEVICE_KERNEL_INTEGRATOR_SHADE_LIGHT_FORWARD
@ DEVICE_KERNEL_INTEGRATOR_SHADE_VOLUME_RAY_MARCHING
@ DEVICE_KERNEL_ADAPTIVE_SAMPLING_CONVERGENCE_FILTER_X
@ DEVICE_KERNEL_INTEGRATOR_SHADE_LIGHT_NEE
@ DEVICE_KERNEL_CRYPTOMATTE_POSTPROCESS
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_INIT_FROM_BAKE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_VOLUME
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_SHADOW
@ DEVICE_KERNEL_INTEGRATOR_INTERSECT_CLOSEST
@ DEVICE_KERNEL_INTEGRATOR_SHADE_BACKGROUND
@ DEVICE_KERNEL_VOLUME_GUIDING_FILTER_Y
@ DEVICE_KERNEL_PREFIX_SUM
#define DCHECK_GE(a, b)
Definition log.h:142
#define LOG_DEBUG
Definition log.h:107
#define DCHECK_LE(a, b)
Definition log.h:147
#define DCHECK_GT(a, b)
Definition log.h:145
#define LOG_IS_ON(level)
Definition log.h:113
#define LOG_FATAL
Definition log.h:99
@ LOG_LEVEL_TRACE
Definition log.h:27
#define LOG_WARNING
Definition log.h:103
#define LOG_INFO
Definition log.h:106
#define LOG_TRACE
Definition log.h:108
PassMode
Definition pass.h:20
@ DENOISED
Definition pass.h:22
static CCL_NAMESPACE_BEGIN size_t estimate_single_state_size(const uint64_t kernel_features, const int volume_stack_size)
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
#define min(a, b)
Definition sort.cc:38
string string_human_readable_size(size_t size)
Definition string.cpp:333
int num_queued[DEVICE_GPU_KERNEL_INTEGRATOR_NUM]
Definition state.h:101
ccl_global int * next_shadow_path_index
Definition state.h:331
ccl_global int * next_main_path_index
Definition state.h:334
ccl_global int * sort_partition_key_offsets
Definition state.h:337
uint sort_partition_divisor
Definition state.h:340
ccl_global int * sort_key_counter[DEVICE_GPU_KERNEL_INTEGRATOR_NUM]
Definition state.h:328
uint64_t device_ptr
Definition types_base.h:44
ccl_device_inline size_t divide_up(const size_t x, const size_t y)
Definition types_base.h:54