Blender V5.3
kernel/device/gpu/kernel.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5/* Common GPU kernels. */
6
10
11#include "kernel/sample/lcg.h"
12
13/* Include constant tables before entering Metal's context class scope (context_begin.h) */
14#include "kernel/tables.h"
15
16#ifdef __KERNEL_METAL__
18#elif defined(__KERNEL_ONEAPI__)
20#endif
21
23
27
42
43#include "kernel/bake/bake.h"
44
47
48#ifdef __KERNEL_METAL__
50#elif defined(__KERNEL_ONEAPI__)
52#endif
53
54#include "kernel/film/read.h"
55
56/* --------------------------------------------------------------------
57 * Integrator.
58 */
59
61 ccl_gpu_kernel_signature(integrator_reset, const int num_states)
62{
63 const int state = ccl_gpu_global_id_x();
64
66 INTEGRATOR_STATE_WRITE(state, path, queued_kernel) = 0;
67 INTEGRATOR_STATE_WRITE(state, shadow_path, queued_kernel) = 0;
68 }
69}
71
75 const int num_tiles,
80{
81 const int work_index = ccl_gpu_global_id_x();
82
83 if (tiles) {
84 if (work_index >= max_tile_work_size * num_tiles) {
85 return;
86 }
87
88 const int tile_index = work_index / max_tile_work_size;
89 const int tile_work_index = work_index - tile_index * max_tile_work_size;
90
92
93 if (tile_work_index >= tile->work_size) {
94 return;
95 }
96
97 const int state = tile->path_index_offset + tile_work_index;
98
99 uint x, y, sample;
101
104 }
105 else {
106 if (work_index >= num_active_paths) {
107 return;
108 }
109
110 const int state = (path_index_array) ? path_index_array[work_index] : work_index;
111
113 integrator_init_from_camera(nullptr, state, nullptr, render_buffer, 0, 0, 0));
114 }
115}
117
121 const int num_tiles,
123 const int max_tile_work_size)
124{
125 const int work_index = ccl_gpu_global_id_x();
126
127 if (work_index >= max_tile_work_size * num_tiles) {
128 return;
129 }
130
131 const int tile_index = work_index / max_tile_work_size;
133
135
136 if (tile_work_index >= tile->work_size) {
137 return;
138 }
139
140 const int state = tile->path_index_offset + tile_work_index;
141
142 uint x, y, sample;
144
147}
149
150#if !defined(__KERNEL_HIPRT__)
151
152/* Intersection kernels need access to the kernel handler for specialization constants to work
153 * properly. */
154# ifdef __KERNEL_ONEAPI__
156# endif
157
160 const ccl_global int *path_index_array,
162 const int work_size)
163{
164 const int global_index = ccl_gpu_global_id_x();
165
167 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
169 }
170}
172
175 const ccl_global int *path_index_array,
176 const int work_size)
177{
178 const int global_index = ccl_gpu_global_id_x();
179
180 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
181 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
183 }
184}
186
189 const ccl_global int *path_index_array,
190 const int work_size)
191{
192 const int global_index = ccl_gpu_global_id_x();
193
194 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
195 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
197 }
198}
200
203 const ccl_global int *path_index_array,
204 const int work_size)
205{
206# ifdef __VOLUME__
207 const int global_index = ccl_gpu_global_id_x();
208
209 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
210 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
212 }
213# endif
214}
216
219 const ccl_global int *path_index_array,
220 const int work_size)
221{
222 const int global_index = ccl_gpu_global_id_x();
223
224 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
225 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
227 }
228}
230
233 const ccl_global int *path_index_array,
234 const int work_size)
235{
236# ifdef __MNEE__
237 const int global_index = ccl_gpu_global_id_x();
238
239 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
240 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
242 }
243# endif
244}
246
247# ifdef __KERNEL_ONEAPI__
249# endif
250
251#endif
252
255 const ccl_global int *path_index_array,
257 const int work_size)
258{
259 const int global_index = ccl_gpu_global_id_x();
260
261 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
262 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
264 }
265}
267
270 const ccl_global int *path_index_array,
272 const int work_size)
273{
274 const int global_index = ccl_gpu_global_id_x();
275
276 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
277 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
279 }
280}
282
285 const ccl_global int *path_index_array,
287 const int work_size)
288{
289 const int global_index = ccl_gpu_global_id_x();
290
291 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
292 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
294 }
295}
297
300 const ccl_global int *path_index_array,
302 const int work_size)
303{
304 const int global_index = ccl_gpu_global_id_x();
305
306 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
307 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
309 }
310}
312
315 const ccl_global int *path_index_array,
317 const int work_size)
318{
319 const int global_index = ccl_gpu_global_id_x();
320
321 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
322 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
324 }
325}
327
328#if defined(__KERNEL_METAL_APPLE__) && defined(__KERNEL_METALRT__)
329constant int __dummy_constant [[function_constant(Kernel_DummyConstant)]];
330#endif
331
332#if !defined(__KERNEL_HIPRT__)
333
334/* Kernels using intersections need access to the kernel handler for specialization constants to
335 * work properly. */
336# ifdef __KERNEL_ONEAPI__
338# endif
339
342 const ccl_global int *path_index_array,
344 const int work_size)
345{
346 const int global_index = ccl_gpu_global_id_x();
347
348 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
349 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
350
351# if defined(__KERNEL_METAL_APPLE__) && defined(__KERNEL_METALRT__)
352 KernelGlobals kg = nullptr;
353 /* Workaround Ambient Occlusion and Bevel nodes not working with Metal.
354 * Dummy offset should not affect result, but somehow fixes bug! */
355 kg += __dummy_constant;
357# else
359# endif
360 }
361}
363
364# ifdef __KERNEL_ONEAPI__
366# endif
367
368#endif
369
372 const ccl_global int *path_index_array,
374 const int work_size)
375{
376 const int global_index = ccl_gpu_global_id_x();
377
378 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
379 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
381 }
382}
384
387 const ccl_global int *path_index_array,
389 const int work_size)
390{
391 const int global_index = ccl_gpu_global_id_x();
392
393 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
394 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
396 }
397}
399
402 const ccl_global int *path_index_array,
404 const int work_size)
405{
406 const int global_index = ccl_gpu_global_id_x();
407
408 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
409 const int state = (path_index_array) ? path_index_array[global_index] : global_index;
411 }
412}
414
416 ccl_gpu_kernel_signature(integrator_queued_paths_array,
417 const int num_states,
418 ccl_global int *indices,
419 ccl_global int *num_indices,
420 const int kernel_index)
421{
422 ccl_gpu_kernel_lambda(INTEGRATOR_STATE(state, path, queued_kernel) == kernel_index,
423 int kernel_index);
424 ccl_gpu_kernel_lambda_pass.kernel_index = kernel_index;
425
426 gpu_parallel_active_index_array(num_states, indices, num_indices, ccl_gpu_kernel_lambda_pass);
427}
429
431 ccl_gpu_kernel_signature(integrator_queued_shadow_paths_array,
432 const int num_states,
433 ccl_global int *indices,
434 ccl_global int *num_indices,
435 const int kernel_index)
436{
437 ccl_gpu_kernel_lambda(INTEGRATOR_STATE(state, shadow_path, queued_kernel) == kernel_index,
438 int kernel_index);
439 ccl_gpu_kernel_lambda_pass.kernel_index = kernel_index;
440
441 gpu_parallel_active_index_array(num_states, indices, num_indices, ccl_gpu_kernel_lambda_pass);
442}
444
446 ccl_gpu_kernel_signature(integrator_active_paths_array,
447 const int num_states,
448 ccl_global int *indices,
449 ccl_global int *num_indices)
450{
451 ccl_gpu_kernel_lambda(INTEGRATOR_STATE(state, path, queued_kernel) != 0);
452
453 gpu_parallel_active_index_array(num_states, indices, num_indices, ccl_gpu_kernel_lambda_pass);
454}
456
458 ccl_gpu_kernel_signature(integrator_terminated_paths_array,
459 const int num_states,
460 ccl_global int *indices,
461 ccl_global int *num_indices,
462 const int indices_offset)
463{
464 ccl_gpu_kernel_lambda(INTEGRATOR_STATE(state, path, queued_kernel) == 0);
465
467 num_states, indices + indices_offset, num_indices, ccl_gpu_kernel_lambda_pass);
468}
470
472 ccl_gpu_kernel_signature(integrator_terminated_shadow_paths_array,
473 const int num_states,
474 ccl_global int *indices,
475 ccl_global int *num_indices,
476 const int indices_offset)
477{
478 ccl_gpu_kernel_lambda(INTEGRATOR_STATE(state, shadow_path, queued_kernel) == 0);
479
481 num_states, indices + indices_offset, num_indices, ccl_gpu_kernel_lambda_pass);
482}
484
486 ccl_gpu_kernel_signature(integrator_sorted_paths_array,
487 const int num_states,
488 const int num_states_limit,
489 ccl_global int *indices,
490 ccl_global int *num_indices,
491 ccl_global int *key_counter,
492 ccl_global int *key_prefix_sum,
493 const int kernel_index)
494{
495 ccl_gpu_kernel_lambda((INTEGRATOR_STATE(state, path, queued_kernel) == kernel_index) ?
496 INTEGRATOR_STATE(state, path, shader_sort_key) :
498 int kernel_index);
499 ccl_gpu_kernel_lambda_pass.kernel_index = kernel_index;
500
501 const uint state_index = ccl_gpu_global_id_x();
504 num_states_limit,
505 indices,
506 num_indices,
507 key_counter,
508 key_prefix_sum,
509 ccl_gpu_kernel_lambda_pass);
510}
512
513/* oneAPI Verizon needs the local_mem accessor in the arguments. */
514#ifdef __KERNEL_ONEAPI__
516 ccl_gpu_kernel_signature(integrator_sort_bucket_pass,
517 const int num_states,
518 const int partition_size,
519 const int num_states_limit,
520 ccl_global int *indices,
521 const int kernel_index,
522 sycl::local_accessor<int> &local_mem)
523#else
525 ccl_gpu_kernel_signature(integrator_sort_bucket_pass,
526 const int num_states,
527 const int partition_size,
528 const int num_states_limit,
529 ccl_global int *indices,
530 const int kernel_index)
531#endif
532{
533 ccl_global ushort *d_queued_kernel = (ccl_global ushort *)
534 kernel_integrator_state.path.queued_kernel;
535 ccl_global uint *d_shader_sort_key = (ccl_global uint *)
536 kernel_integrator_state.path.shader_sort_key;
537 ccl_global int *key_offsets = (ccl_global int *)
538 kernel_integrator_state.sort_partition_key_offsets;
539
540#ifdef __KERNEL_ONEAPI__
541 /* There is no difference here between different access decorations, as we are requesting
542 * a raw pointer immediately, so the simplest decoration option is used (no decoration). */
543 ccl_gpu_shared int *threadgroup_array =
544 local_mem.get_multi_ptr<sycl::access::decorated::no>().get();
545#endif
546
548 partition_size,
549 kernel_data.max_shaders,
550 kernel_index,
551 d_queued_kernel,
552 d_shader_sort_key,
553 key_offsets);
554}
556
557/* oneAPI version needs the local_mem accessor in the arguments. */
558#ifdef __KERNEL_ONEAPI__
560 ccl_gpu_kernel_signature(integrator_sort_write_pass,
561 const int num_states,
562 const int partition_size,
563 const int num_states_limit,
564 ccl_global int *indices,
565 const int kernel_index,
566 sycl::local_accessor<int> &local_mem)
567#else
569 ccl_gpu_kernel_signature(integrator_sort_write_pass,
570 const int num_states,
571 const int partition_size,
572 const int num_states_limit,
573 ccl_global int *indices,
574 const int kernel_index)
575#endif
576
577{
578 ccl_global ushort *d_queued_kernel = (ccl_global ushort *)
579 kernel_integrator_state.path.queued_kernel;
580 ccl_global uint *d_shader_sort_key = (ccl_global uint *)
581 kernel_integrator_state.path.shader_sort_key;
582 ccl_global int *key_offsets = (ccl_global int *)
583 kernel_integrator_state.sort_partition_key_offsets;
584
585#ifdef __KERNEL_ONEAPI__
586 /* There is no difference here between different access decorations, as we are requesting
587 * a raw pointer immediately, so the simplest decoration option is used (no decoration). */
588 ccl_gpu_shared int *threadgroup_array =
589 local_mem.get_multi_ptr<sycl::access::decorated::no>().get();
590#endif
591
593 partition_size,
594 num_states_limit,
595 indices,
596 kernel_data.max_shaders,
597 kernel_index,
598 d_queued_kernel,
599 d_shader_sort_key,
600 key_offsets);
601}
603
605 ccl_gpu_kernel_signature(integrator_compact_paths_array,
606 const int num_states,
607 ccl_global int *indices,
608 ccl_global int *num_indices,
609 const int num_active_paths)
610{
612 (INTEGRATOR_STATE(state, path, queued_kernel) != 0),
613 int num_active_paths);
614 ccl_gpu_kernel_lambda_pass.num_active_paths = num_active_paths;
615
616 gpu_parallel_active_index_array(num_states, indices, num_indices, ccl_gpu_kernel_lambda_pass);
617}
619
621 ccl_gpu_kernel_signature(integrator_compact_states,
622 const ccl_global int *active_terminated_states,
623 const int active_states_offset,
624 const int terminated_states_offset,
625 const int work_size)
626{
627 const int global_index = ccl_gpu_global_id_x();
628
629 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
630 const int from_state = active_terminated_states[active_states_offset + global_index];
631 const int to_state = active_terminated_states[terminated_states_offset + global_index];
632
633 ccl_gpu_kernel_call(integrator_state_move(nullptr, to_state, from_state));
634 }
635}
637
639 ccl_gpu_kernel_signature(integrator_compact_shadow_paths_array,
640 const int num_states,
641 ccl_global int *indices,
642 ccl_global int *num_indices,
643 const int num_active_paths)
644{
646 (INTEGRATOR_STATE(state, shadow_path, queued_kernel) != 0),
647 int num_active_paths);
648 ccl_gpu_kernel_lambda_pass.num_active_paths = num_active_paths;
649
650 gpu_parallel_active_index_array(num_states, indices, num_indices, ccl_gpu_kernel_lambda_pass);
651}
653
655 ccl_gpu_kernel_signature(integrator_compact_shadow_states,
656 const ccl_global int *active_terminated_states,
657 const int active_states_offset,
658 const int terminated_states_offset,
659 const int work_size)
660{
661 const int global_index = ccl_gpu_global_id_x();
662
663 if (ccl_gpu_kernel_within_bounds(global_index, work_size)) {
664 const int from_state = active_terminated_states[active_states_offset + global_index];
665 const int to_state = active_terminated_states[terminated_states_offset + global_index];
666
667 ccl_gpu_kernel_call(integrator_shadow_state_move(nullptr, to_state, from_state));
668 }
669}
671
673 prefix_sum, ccl_global int *counter, ccl_global int *prefix_sum, const int num_values)
674{
675 gpu_parallel_prefix_sum(ccl_gpu_global_id_x(), counter, prefix_sum, num_values);
676}
678
679/* --------------------------------------------------------------------
680 * Adaptive sampling.
681 */
682
686 const int sx,
687 const int sy,
688 const int sw,
689 const int sh,
690 const float threshold,
691 const int reset,
692 const int offset,
693 const int stride,
694 ccl_global uint *num_active_pixels)
695{
696 const int work_index = ccl_gpu_global_id_x();
697 const int y = work_index / sw;
698 const int x = work_index - y * sw;
699 const int lane_id = ccl_gpu_thread_idx_x % ccl_gpu_warp_size;
700
701 bool converged = true;
702
703 if (x < sw && y < sh) {
705 nullptr, render_buffer, sx + x, sy + y, threshold, reset, offset, stride));
706 }
707
708#ifdef __KERNEL_ONEAPI__
709 const sycl::nd_item<1> &item_id = sycl::ext::oneapi::this_work_item::get_nd_item<1>();
710 const uint num_active_pixels_in_warp = sycl::inclusive_scan_over_group(
711 item_id.get_sub_group(), static_cast<uint>(!converged), std::plus<>());
712 if (lane_id == item_id.get_sub_group().get_local_range()[0] - 1) {
713 atomic_fetch_and_add_uint32(num_active_pixels, num_active_pixels_in_warp);
714 }
715#else
716 /* NOTE: All threads specified in the mask must execute the intrinsic. */
717 const auto num_active_pixels_mask = ccl_gpu_ballot(!converged);
718 if (lane_id == 0) {
719 atomic_fetch_and_add_uint32(num_active_pixels, popcount(num_active_pixels_mask));
720 }
721#endif
722}
724
728 const int sx,
729 const int sy,
730 const int sw,
731 const int sh,
732 const int offset,
733 const int stride)
734{
735 const int y = ccl_gpu_global_id_x();
736
737 if (y < sh) {
739 film_adaptive_sampling_filter_x(nullptr, render_buffer, sy + y, sx, sw, offset, stride));
740 }
741}
743
747 const int sx,
748 const int sy,
749 const int sw,
750 const int sh,
751 const int offset,
752 const int stride)
753{
754 const int x = ccl_gpu_global_id_x();
755
756 if (x < sw) {
758 film_adaptive_sampling_filter_y(nullptr, render_buffer, sx + x, sy, sh, offset, stride));
759 }
760}
762
763/* --------------------------------------------------------------------
764 * Cryptomatte.
765 */
766
770 const int num_pixels)
771{
772 const int pixel_index = ccl_gpu_global_id_x();
773
774 if (pixel_index < num_pixels) {
776 }
777}
779
780/* --------------------------------------------------------------------
781 * Film.
782 */
783
784ccl_device_inline void kernel_gpu_film_convert_half_write(ccl_global uchar4 *rgba,
785 const int rgba_offset,
786 const int rgba_stride,
787 const int x,
788 const int y,
789 const half4 half_pixel)
790{
791 /* Work around HIP issue with half float display, see #92972. */
792#ifdef __KERNEL_HIP__
793 ccl_global half *out = ((ccl_global half *)rgba) + (rgba_offset + y * rgba_stride + x) * 4;
794 out[0] = half_pixel.x;
795 out[1] = half_pixel.y;
796 out[2] = half_pixel.z;
797 out[3] = half_pixel.w;
798#else
799 ccl_global half4 *out = ((ccl_global half4 *)rgba) + rgba_offset + y * rgba_stride + x;
800 *out = half_pixel;
801#endif
802}
803
804#ifdef __KERNEL_METAL__
805
806/* Fetch into a local variable on Metal - there is minimal overhead. Templating the
807 * film_get_pass_pixel_... functions works on MSL, but not on other compilers. */
808# define FILM_GET_PASS_PIXEL_F32(variant, input_channel_count) \
809 float local_pixel[4]; \
810 film_get_pass_pixel_##variant(&kfilm_convert, buffer, local_pixel); \
811 if (input_channel_count >= 1) { \
812 pixel[0] = local_pixel[0]; \
813 } \
814 if (input_channel_count >= 2) { \
815 pixel[1] = local_pixel[1]; \
816 } \
817 if (input_channel_count >= 3) { \
818 pixel[2] = local_pixel[2]; \
819 } \
820 if (input_channel_count >= 4) { \
821 pixel[3] = local_pixel[3]; \
822 }
823
824#else
825
826# define FILM_GET_PASS_PIXEL_F32(variant, input_channel_count) \
827 film_get_pass_pixel_##variant(&kfilm_convert, buffer, pixel);
828
829#endif
830
831#define KERNEL_FILM_CONVERT_VARIANT(variant, input_channel_count) \
832 ccl_gpu_kernel(GPU_KERNEL_BLOCK_NUM_THREADS, GPU_KERNEL_MAX_REGISTERS) \
833 ccl_gpu_kernel_signature(film_convert_##variant, \
834 const KernelFilmConvert kfilm_convert, \
835 ccl_global float *pixels, \
836 ccl_global float *render_buffer, \
837 int num_pixels, \
838 int width, \
839 int offset, \
840 int stride, \
841 int channel_offset, \
842 int rgba_offset, \
843 int rgba_stride) \
844 { \
845 const int render_pixel_index = ccl_gpu_global_id_x(); \
846 if (render_pixel_index >= num_pixels) { \
847 return; \
848 } \
849\
850 const int x = render_pixel_index % width; \
851 const int y = render_pixel_index / width; \
852\
853 const uint64_t buffer_pixel_index = x + y * stride; \
854 ccl_global const float *buffer = render_buffer + offset + \
855 buffer_pixel_index * kfilm_convert.pass_stride; \
856\
857 ccl_global float *pixel = pixels + channel_offset + \
858 (render_pixel_index + rgba_offset) * kfilm_convert.pixel_stride; \
859\
860 FILM_GET_PASS_PIXEL_F32(variant, input_channel_count); \
861 } \
862 ccl_gpu_kernel_postfix \
863\
864 ccl_gpu_kernel(GPU_KERNEL_BLOCK_NUM_THREADS, GPU_KERNEL_MAX_REGISTERS) \
865 ccl_gpu_kernel_signature(film_convert_##variant##_half_rgba, \
866 const KernelFilmConvert kfilm_convert, \
867 ccl_global uchar4 *rgba, \
868 ccl_global float *render_buffer, \
869 int num_pixels, \
870 int width, \
871 int offset, \
872 int stride, \
873 int rgba_offset, \
874 int rgba_stride) \
875 { \
876 const int render_pixel_index = ccl_gpu_global_id_x(); \
877 if (render_pixel_index >= num_pixels) { \
878 return; \
879 } \
880\
881 const int x = render_pixel_index % width; \
882 const int y = render_pixel_index / width; \
883\
884 const uint64_t buffer_pixel_index = x + y * stride; \
885 ccl_global const float *buffer = render_buffer + offset + \
886 buffer_pixel_index * kfilm_convert.pass_stride; \
887\
888 float pixel[4]; \
889 film_get_pass_pixel_##variant(&kfilm_convert, buffer, pixel); \
890\
891 if (input_channel_count == 1) { \
892 pixel[1] = pixel[2] = pixel[0]; \
893 } \
894 if (input_channel_count <= 3) { \
895 pixel[3] = 1.0f; \
896 } \
897\
898 film_apply_pass_pixel_overlays_rgba(&kfilm_convert, buffer, pixel); \
899\
900 const half4 half_pixel = float4_to_half4_display( \
901 make_float4(pixel[0], pixel[1], pixel[2], pixel[3])); \
902 kernel_gpu_film_convert_half_write(rgba, rgba_offset, rgba_stride, x, y, half_pixel); \
903 } \
904 ccl_gpu_kernel_postfix
905
906/* 1 channel inputs */
909KERNEL_FILM_CONVERT_VARIANT(volume_majorant, 1)
910KERNEL_FILM_CONVERT_VARIANT(sample_count, 1)
912
913/* 3 channel inputs */
914KERNEL_FILM_CONVERT_VARIANT(light_path, 3)
917
918/* 4 channel inputs */
920KERNEL_FILM_CONVERT_VARIANT(cryptomatte, 4)
921KERNEL_FILM_CONVERT_VARIANT(shadow_catcher, 4)
922KERNEL_FILM_CONVERT_VARIANT(shadow_catcher_matte_with_shadow, 4)
925
926#undef KERNEL_FILM_CONVERT_VARIANT
927
928/* --------------------------------------------------------------------
929 * Shader evaluation.
930 */
931
932/* Displacement */
933
937 ccl_global float *output,
938 ccl_global uint *cache_miss,
939 const int offset,
940 const int work_size)
941{
942 int i = ccl_gpu_global_id_x();
943 if (i < work_size) {
944 ccl_gpu_kernel_call(kernel_displace_evaluate(nullptr, input, output, cache_miss, offset + i));
945 }
946}
948
949/* Background */
950
954 ccl_global float *output,
955 ccl_global uint *cache_miss,
956 const int offset,
957 const int work_size)
958{
959 int i = ccl_gpu_global_id_x();
960 if (i < work_size) {
962 kernel_background_evaluate(nullptr, input, output, cache_miss, offset + i));
963 }
964}
966
967/* Curve Shadow Transparency */
968
972 ccl_global float *output,
973 ccl_global uint *cache_miss,
974 const int offset,
975 const int work_size)
976{
977 int i = ccl_gpu_global_id_x();
978 if (i < work_size) {
980 kernel_curve_shadow_transparency_evaluate(nullptr, input, output, cache_miss, offset + i));
981 }
982}
984
985/* Volume Density. */
986
990 ccl_global float *output,
991 ccl_global uint *cache_miss,
992 const int offset,
993 const int work_size)
994{
995 int i = ccl_gpu_global_id_x();
996 if (i < work_size) {
998 kernel_volume_density_evaluate(nullptr, input, output, cache_miss, offset + i));
999 }
1000}
1002
1003/* --------------------------------------------------------------------
1004 * Denoising.
1005 */
1006
1008 ccl_gpu_kernel_signature(filter_color_preprocess,
1010 const int full_x,
1011 const int full_y,
1012 const int width,
1013 const int height,
1014 const int offset,
1015 const int stride,
1016 const int pass_stride,
1017 const int pass_denoised)
1018{
1019 const int work_index = ccl_gpu_global_id_x();
1020 const int y = work_index / width;
1021 const int x = work_index - y * width;
1022
1023 if (x >= width || y >= height) {
1024 return;
1025 }
1026
1027 const uint64_t render_pixel_index = offset + (x + full_x) + (y + full_y) * stride;
1028 ccl_global float *buffer = render_buffer + render_pixel_index * pass_stride;
1029
1030 ccl_global float *color_out = buffer + pass_denoised;
1031 color_out[0] = clamp(color_out[0], 0.0f, 10000.0f);
1032 color_out[1] = clamp(color_out[1], 0.0f, 10000.0f);
1033 color_out[2] = clamp(color_out[2], 0.0f, 10000.0f);
1034}
1036
1038 ccl_gpu_kernel_signature(filter_color_preprocess_to_surface,
1039 const uint64_t color_surface,
1041 const int full_x,
1042 const int full_y,
1043 const int width,
1044 const int height,
1045 const int offset,
1046 const int stride,
1047 const int pass_stride,
1048 const int pass_denoised,
1049 const int num_components)
1050{
1051#ifdef __KERNEL_CUDA__
1052 const int work_index = ccl_gpu_global_id_x();
1053 const int y = work_index / width;
1054 const int x = work_index - y * width;
1055
1056 if (x >= width || y >= height) {
1057 return;
1058 }
1059
1060 const uint64_t render_pixel_index = offset + (x + full_x) + (y + full_y) * stride;
1061 ccl_global float *buffer = render_buffer + render_pixel_index * pass_stride;
1062
1063 ccl_global float *denoised_pixel = buffer + pass_denoised;
1064
1065 float4 color_value;
1066 color_value.x = denoised_pixel[0];
1067 color_value.y = denoised_pixel[1];
1068 color_value.z = denoised_pixel[2];
1069 color_value.w = num_components > 3 ? denoised_pixel[3] : 1.0f;
1070
1071 surf2Dwrite(color_value, color_surface, x * sizeof(float4), y);
1072#endif
1073}
1075
1077 ccl_gpu_kernel_signature(filter_guiding_preprocess,
1078 ccl_global float *guiding_buffer,
1079 const int guiding_pass_stride,
1080 const int guiding_pass_albedo,
1081 const int guiding_pass_normal,
1082 const int guiding_pass_flow,
1083 const ccl_global float *render_buffer,
1084 const int render_offset,
1085 const int render_stride,
1086 const int render_pass_stride,
1087 const int render_pass_sample_count,
1088 const int render_pass_denoising_albedo,
1089 const int render_pass_denoising_normal,
1090 const int render_pass_motion,
1091 const int full_x,
1092 const int full_y,
1093 const int width,
1094 const int height,
1095 const int num_samples)
1096{
1097 const int work_index = ccl_gpu_global_id_x();
1098 const int y = work_index / width;
1099 const int x = work_index - y * width;
1100
1101 if (x >= width || y >= height) {
1102 return;
1103 }
1104
1105 const uint64_t render_pixel_index = render_offset + (x + full_x) + (y + full_y) * render_stride;
1106 const ccl_global float *buffer = render_buffer + render_pixel_index * render_pass_stride;
1107
1108 const uint64_t guiding_pixel_index = x + y * width;
1109 ccl_global float *guiding_pixel = guiding_buffer + guiding_pixel_index * guiding_pass_stride;
1110
1111 float pixel_scale;
1112 if (render_pass_sample_count == PASS_UNUSED) {
1113 pixel_scale = 1.0f / num_samples;
1114 }
1115 else {
1116 pixel_scale = 1.0f / __float_as_uint(buffer[render_pass_sample_count]);
1117 }
1118
1119 /* Albedo pass. */
1120 if (guiding_pass_albedo != PASS_UNUSED) {
1121 kernel_assert(render_pass_denoising_albedo != PASS_UNUSED);
1122
1123 const ccl_global float *albedo_in = buffer + render_pass_denoising_albedo;
1124 ccl_global float *albedo_out = guiding_pixel + guiding_pass_albedo;
1125
1126 albedo_out[0] = albedo_in[0] * pixel_scale;
1127 albedo_out[1] = albedo_in[1] * pixel_scale;
1128 albedo_out[2] = albedo_in[2] * pixel_scale;
1129 }
1130
1131 /* Normal pass. */
1132 if (guiding_pass_normal != PASS_UNUSED) {
1133 kernel_assert(render_pass_denoising_normal != PASS_UNUSED);
1134
1135 const ccl_global float *normal_in = buffer + render_pass_denoising_normal;
1136 ccl_global float *normal_out = guiding_pixel + guiding_pass_normal;
1137
1138 normal_out[0] = normal_in[0] * pixel_scale;
1139 normal_out[1] = normal_in[1] * pixel_scale;
1140 normal_out[2] = normal_in[2] * pixel_scale;
1141 }
1142
1143 /* Flow pass. */
1144 if (guiding_pass_flow != PASS_UNUSED) {
1145 kernel_assert(render_pass_motion != PASS_UNUSED);
1146
1147 const ccl_global float *motion_in = buffer + render_pass_motion;
1148 ccl_global float *flow_out = guiding_pixel + guiding_pass_flow;
1149
1150 flow_out[0] = -motion_in[0] * pixel_scale;
1151 flow_out[1] = -motion_in[1] * pixel_scale;
1152 }
1153}
1155
1157 ccl_gpu_kernel_signature(filter_guiding_preprocess_to_surface,
1158 const uint64_t depth_surface,
1159 const uint64_t albedo_surface,
1160 const uint64_t specular_albedo_surface,
1161 const uint64_t normal_roughness_surface,
1162 const uint64_t motion_surface,
1163 const uint64_t specular_motion_surface,
1164 const ccl_global float *render_buffer,
1165 const int render_offset,
1166 const int render_stride,
1167 const int render_pass_stride,
1168 const int render_pass_sample_count,
1169 const int render_pass_depth,
1170 const int render_pass_albedo,
1171 const int render_pass_specular_albedo,
1172 const int render_pass_normal,
1173 const int render_pass_roughness,
1174 const int render_pass_motion,
1175 const int render_pass_specular_motion,
1176 const int full_x,
1177 const int full_y,
1178 const int width,
1179 const int height,
1180 const int num_samples)
1181{
1182#ifdef __KERNEL_CUDA__
1183 const int work_index = ccl_gpu_global_id_x();
1184 const int y = work_index / width;
1185 const int x = work_index - y * width;
1186
1187 if (x >= width || y >= height) {
1188 return;
1189 }
1190
1191 const uint64_t render_pixel_index = render_offset + (x + full_x) + (y + full_y) * render_stride;
1192 const ccl_global float *buffer = render_buffer + render_pixel_index * render_pass_stride;
1193
1194 float pixel_scale;
1195 if (render_pass_sample_count == PASS_UNUSED) {
1196 pixel_scale = 1.0f / num_samples;
1197 }
1198 else {
1199 pixel_scale = 1.0f / __float_as_uint(buffer[render_pass_sample_count]);
1200 }
1201
1202 /* Depth pass. */
1203 if (render_pass_depth != PASS_UNUSED) {
1204 const ccl_global float *depth_in = buffer + render_pass_depth;
1205
1206 const float depth_value = depth_in[0] * pixel_scale;
1207
1208 surf2Dwrite(depth_value, depth_surface, x * sizeof(float), y);
1209 }
1210
1211 /* Diffuse albedo pass. */
1212 if (render_pass_albedo != PASS_UNUSED) {
1213 const ccl_global float *albedo_in = buffer + render_pass_albedo;
1214
1215 float4 albedo_value;
1216 albedo_value.x = albedo_in[0] * pixel_scale;
1217 albedo_value.y = albedo_in[1] * pixel_scale;
1218 albedo_value.z = albedo_in[2] * pixel_scale;
1219 albedo_value.w = 1.0;
1220
1221 /* Tonemap the albedo with simple reinhard operator. */
1222 albedo_value.x = clamp(albedo_value.x / (1.0f + albedo_value.x), 0.0f, 1.0f);
1223 albedo_value.y = clamp(albedo_value.y / (1.0f + albedo_value.y), 0.0f, 1.0f);
1224 albedo_value.z = clamp(albedo_value.z / (1.0f + albedo_value.z), 0.0f, 1.0f);
1225
1226 surf2Dwrite(albedo_value, albedo_surface, x * sizeof(float4), y);
1227 }
1228
1229 /* Specular albedo pass. */
1230 if (render_pass_specular_albedo != PASS_UNUSED) {
1231 const ccl_global float *albedo_in = buffer + render_pass_specular_albedo;
1232
1233 float4 specular_albedo_value;
1234 specular_albedo_value.x = albedo_in[0] * pixel_scale;
1235 specular_albedo_value.y = albedo_in[1] * pixel_scale;
1236 specular_albedo_value.z = albedo_in[2] * pixel_scale;
1237 specular_albedo_value.w = 1.0;
1238
1239 surf2Dwrite(specular_albedo_value, specular_albedo_surface, x * sizeof(float4), y);
1240 }
1241
1242 /* Normal and roughness pass. */
1243 if (render_pass_normal != PASS_UNUSED && render_pass_roughness != PASS_UNUSED) {
1244 const ccl_global float *normal_in = buffer + render_pass_normal;
1245 const ccl_global float *roughness_in = buffer + render_pass_roughness;
1246
1247 float4 normal_roughness_value;
1248 normal_roughness_value.x = normal_in[0] * pixel_scale;
1249 normal_roughness_value.y = normal_in[1] * pixel_scale;
1250 normal_roughness_value.z = normal_in[2] * pixel_scale;
1251 normal_roughness_value.w = roughness_in[0] * pixel_scale;
1252
1253 surf2Dwrite(normal_roughness_value, normal_roughness_surface, x * sizeof(float4), y);
1254 }
1255
1256 /* Motion pass. */
1257 if (render_pass_motion != PASS_UNUSED) {
1258 const ccl_global float *motion_in = buffer + render_pass_motion;
1259
1260 float2 motion_value;
1261 motion_value.x = motion_in[0] * pixel_scale;
1262 motion_value.y = motion_in[1] * pixel_scale;
1263
1264 surf2Dwrite(motion_value, motion_surface, x * sizeof(float2), y);
1265 }
1266
1267 /* Specular motion pass. */
1268 if (render_pass_specular_motion != PASS_UNUSED) {
1269 const ccl_global float *motion_in = buffer + render_pass_specular_motion;
1270
1271 float2 motion_value;
1272 motion_value.x = motion_in[0] * pixel_scale;
1273 motion_value.y = motion_in[1] * pixel_scale;
1274
1275 surf2Dwrite(motion_value, specular_motion_surface, x * sizeof(float2), y);
1276 }
1277#endif
1278}
1280
1282 ccl_gpu_kernel_signature(filter_guiding_set_fake_albedo,
1283 ccl_global float *guiding_buffer,
1284 const int guiding_pass_stride,
1285 const int guiding_pass_albedo,
1286 const int width,
1287 const int height)
1288{
1289 kernel_assert(guiding_pass_albedo != PASS_UNUSED);
1290
1291 const int work_index = ccl_gpu_global_id_x();
1292 const int y = work_index / width;
1293 const int x = work_index - y * width;
1294
1295 if (x >= width || y >= height) {
1296 return;
1297 }
1298
1299 const uint64_t guiding_pixel_index = x + y * width;
1300 ccl_global float *guiding_pixel = guiding_buffer + guiding_pixel_index * guiding_pass_stride;
1301
1302 ccl_global float *albedo_out = guiding_pixel + guiding_pass_albedo;
1303 albedo_out[0] = 0.5f;
1304 albedo_out[1] = 0.5f;
1305 albedo_out[2] = 0.5f;
1306}
1308
1310 ccl_gpu_kernel_signature(filter_color_postprocess,
1312 const int full_x,
1313 const int full_y,
1314 const int width,
1315 const int height,
1316 const int offset,
1317 const int stride,
1318 const int pass_stride,
1319 const int num_samples,
1320 const int pass_noisy,
1321 const int pass_denoised,
1322 const int pass_sample_count,
1323 const int num_components,
1324 const float upscale_factor)
1325{
1326 const int work_index = ccl_gpu_global_id_x();
1327 const int y = work_index / width;
1328 const int x = work_index - y * width;
1329
1330 if (x >= width || y >= height) {
1331 return;
1332 }
1333
1334 const uint64_t render_pixel_index = offset + (x + full_x) + (y + full_y) * stride;
1335 ccl_global float *buffer = render_buffer + render_pixel_index * pass_stride;
1336
1337 float pixel_scale;
1338 if (pass_sample_count == PASS_UNUSED) {
1339 pixel_scale = num_samples;
1340 }
1341 else if (upscale_factor == 1.0f) {
1342 pixel_scale = __float_as_uint(buffer[pass_sample_count]);
1343 }
1344
1345 ccl_global float *denoised_pixel = buffer + pass_denoised;
1346
1347 if (num_components > 3) {
1348 /* Convert alpha back to transparency. */
1349 denoised_pixel[3] = 1.0f - denoised_pixel[3];
1350 }
1351
1352 if (pass_sample_count == PASS_UNUSED || upscale_factor == 1.0f) {
1353 for (int c = 0; c < 4 && c < num_components; c++) {
1354 denoised_pixel[c] *= pixel_scale;
1355 }
1356 }
1357}
1359
1361 ccl_gpu_kernel_signature(filter_color_postprocess_from_surface,
1362 const uint64_t color_surface,
1364 const int full_x,
1365 const int full_y,
1366 const int width,
1367 const int height,
1368 const int offset,
1369 const int stride,
1370 const int pass_stride,
1371 const int num_samples,
1372 const int pass_noisy,
1373 const int pass_denoised,
1374 const int pass_sample_count,
1375 const int num_components,
1376 const float upscale_factor)
1377{
1378#ifdef __KERNEL_CUDA__
1379 const int work_index = ccl_gpu_global_id_x();
1380 const int y = work_index / width;
1381 const int x = work_index - y * width;
1382
1383 if (x >= width || y >= height) {
1384 return;
1385 }
1386
1387 const uint64_t render_pixel_index = offset + (x + full_x) + (y + full_y) * stride;
1388 ccl_global float *buffer = render_buffer + render_pixel_index * pass_stride;
1389
1390 float pixel_scale;
1391 if (pass_sample_count == PASS_UNUSED) {
1392 pixel_scale = num_samples;
1393 }
1394 else if (upscale_factor == 1.0f) {
1395 pixel_scale = __float_as_uint(buffer[pass_sample_count]);
1396 }
1397
1398 ccl_global float *denoised_pixel = buffer + pass_denoised;
1399
1400 float4 color_value;
1401 surf2Dread(&color_value, color_surface, x * sizeof(float4), y);
1402
1403 denoised_pixel[0] = color_value.x;
1404 denoised_pixel[1] = color_value.y;
1405 denoised_pixel[2] = color_value.z;
1406 if (num_components > 3) {
1407 /* Convert alpha back to transparency. */
1408 denoised_pixel[3] = 1.0f - color_value.w;
1409 }
1410
1411 if (pass_sample_count == PASS_UNUSED || upscale_factor == 1.0f) {
1412 for (int c = 0; c < 4 && c < num_components; c++) {
1413 denoised_pixel[c] *= pixel_scale;
1414 }
1415 }
1416#endif
1417}
1419
1421 ccl_gpu_kernel_signature(filter_color_flip_y,
1423 const int full_x,
1424 const int full_y,
1425 const int width,
1426 const int height,
1427 const int offset,
1428 const int stride,
1429 const int pass_stride,
1430 const int pass_denoised,
1431 const int num_components)
1432{
1433 const int work_index = ccl_gpu_global_id_x();
1434 const int y = work_index / width;
1435 const int x = work_index - y * width;
1436
1437 if (x >= width || y >= height / 2) {
1438 return;
1439 }
1440
1441 const uint64_t render_pixel_index = offset + (x + full_x) + (y + full_y) * stride;
1442 ccl_global float *buffer = render_buffer + render_pixel_index * pass_stride + pass_denoised;
1443 ccl_global float *buffer_flipped = buffer + (height - 1 - y * 2) * stride * pass_stride;
1444
1445 float temp[4];
1446 for (int c = 0; c < 4 && c < num_components; c++) {
1447 temp[c] = buffer[c];
1448 buffer[c] = buffer_flipped[c];
1449 buffer_flipped[c] = temp[c];
1450 }
1451}
1453
1454/* --------------------------------------------------------------------
1455 * Shadow catcher.
1456 */
1457
1459 ccl_gpu_kernel_signature(integrator_shadow_catcher_count_possible_splits,
1460 const int num_states,
1461 ccl_global uint *num_possible_splits)
1462{
1463 const int state = ccl_gpu_global_id_x();
1464 const int lane_id = ccl_gpu_thread_idx_x % ccl_gpu_warp_size;
1465
1466 bool can_split = false;
1467
1468 if (state < num_states) {
1470 }
1471
1472#ifdef __KERNEL_ONEAPI__
1473 const sycl::nd_item<1> &item_id = sycl::ext::oneapi::this_work_item::get_nd_item<1>();
1474 const uint num_possible_splits_in_warp = sycl::inclusive_scan_over_group(
1475 item_id.get_sub_group(), static_cast<uint>(can_split), std::plus<>());
1476 if (lane_id == item_id.get_sub_group().get_local_range()[0] - 1) {
1477 atomic_fetch_and_add_uint32(num_possible_splits, num_possible_splits_in_warp);
1478 }
1479#else
1480 /* NOTE: All threads specified in the mask must execute the intrinsic. */
1481 const auto can_split_mask = ccl_gpu_ballot(can_split);
1482 if (lane_id == 0) {
1483 atomic_fetch_and_add_uint32(num_possible_splits, popcount(can_split_mask));
1484 }
1485#endif
1486}
1488
1489/* --------------------------------------------------------------------
1490 * Volume Scattering Probability Guiding.
1491 */
1492
1496 const int sx,
1497 const int sy,
1498 const int sw,
1499 const int sh,
1500 const int offset,
1501 const int stride)
1502{
1503 const int work_index = ccl_gpu_global_id_x();
1504 const int y = work_index / sw;
1505 const int x = work_index % sw;
1506
1507 if (y < sh) {
1509 nullptr, render_buffer, sy + y, sx + x, sx, sx + sw, offset, stride));
1510 }
1511}
1513
1517 const int sx,
1518 const int sy,
1519 const int sw,
1520 const int sh,
1521 const int offset,
1522 const int stride)
1523{
1524 const int x = ccl_gpu_global_id_x();
1525
1526 if (x < sw) {
1528 volume_guiding_filter_y(nullptr, render_buffer, sx + x, sy, sy + sh, offset, stride));
1529 }
1530}
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
#define GPU_PARALLEL_SORTED_INDEX_INACTIVE_KEY
Definition block_sizes.h:17
#define GPU_PARALLEL_SORTED_INDEX_DEFAULT_BLOCK_SIZE
Definition block_sizes.h:14
#define GPU_PARALLEL_PREFIX_SUM_DEFAULT_BLOCK_SIZE
Definition block_sizes.h:13
#define GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE
Definition block_sizes.h:12
#define GPU_PARALLEL_SORT_BLOCK_SIZE
Definition block_sizes.h:18
unsigned long long int uint64_t
#define output
void reset()
Definition half.h:29
ccl_device_inline void film_cryptomatte_post(KernelGlobals kg, ccl_global float *render_buffer, const int pixel_index)
#define kernel_assert(cond)
#define kernel_data
#define PASS_UNUSED
#define ccl_device_inline
#define ccl_global
#define ccl_gpu_thread_idx_x
#define ccl_gpu_global_id_x()
#define ccl_gpu_warp_size
#define ccl_gpu_shared
#define ccl_gpu_ballot(predicate)
#define kernel_integrator_state
#define __float_as_uint(x)
@ Kernel_DummyConstant
constexpr T clamp(T, U, U) RET
ccl_device bool integrator_init_from_bake(KernelGlobals kg, IntegratorState state, const ccl_global KernelWorkTile *ccl_restrict tile, ccl_global float *render_buffer, const int x, const int y, const int scheduled_sample)
ccl_device bool integrator_init_from_camera(KernelGlobals kg, IntegratorState state, const ccl_global KernelWorkTile *ccl_restrict tile, ccl_global float *render_buffer, const int x_, const int y_, const int scheduled_sample)
ccl_device void integrator_intersect_closest(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
CCL_NAMESPACE_BEGIN ccl_device void integrator_intersect_dedicated_light(KernelGlobals kg, IntegratorState state)
CCL_NAMESPACE_BEGIN ccl_device void integrator_intersect_mnee(KernelGlobals kg, IntegratorState state)
ccl_device void integrator_intersect_shadow(KernelGlobals kg, IntegratorShadowState state)
CCL_NAMESPACE_BEGIN ccl_device void integrator_intersect_subsurface(KernelGlobals kg, IntegratorState state)
ccl_device void integrator_intersect_volume_stack(KernelGlobals kg, IntegratorState state)
ccl_device void kernel_volume_density_evaluate(KernelGlobals kg, ccl_global const KernelShaderEvalInput *input, ccl_global float *output, ccl_global uint *cache_miss, const int offset)
ccl_device void kernel_background_evaluate(KernelGlobals kg, const ccl_global KernelShaderEvalInput *input, ccl_global float *output, ccl_global uint *cache_miss, const int offset)
ccl_device void kernel_curve_shadow_transparency_evaluate(KernelGlobals kg, const ccl_global KernelShaderEvalInput *input, ccl_global float *output, ccl_global uint *cache_miss, const int offset)
CCL_NAMESPACE_BEGIN ccl_device void kernel_displace_evaluate(KernelGlobals kg, const ccl_global KernelShaderEvalInput *input, ccl_global float *output, ccl_global uint *cache_miss, const int offset)
#define ccl_gpu_kernel(block_num_threads, thread_num_registers)
#define GPU_KERNEL_MAX_REGISTERS
#define GPU_KERNEL_BLOCK_NUM_THREADS
#define ccl_gpu_kernel_within_bounds(i, n)
#define ccl_gpu_kernel_call(x)
#define ccl_gpu_kernel_postfix
#define ccl_gpu_kernel_threads(block_num_threads)
#define ccl_gpu_kernel_lambda(func,...)
#define ccl_gpu_kernel_signature(name,...)
const int tile_work_index
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int const ccl_global int const int num_active_paths
const int num_states
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int num_tiles
#define KERNEL_FILM_CONVERT_VARIANT(variant, input_channel_count)
ccl_gpu_kernel_postfix ccl_global KernelWorkTile * tiles
ccl_gpu_kernel_postfix const ccl_global int ccl_global float const int work_size
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int max_tile_work_size
const int tile_index
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float const int const ccl_global int * path_index_array
const int state
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int ccl_global float * render_buffer
const ccl_global KernelWorkTile * tile
ccl_device void film_adaptive_sampling_filter_y(KernelGlobals kg, ccl_global float *render_buffer, const int x, const int start_y, const int height, const int offset, const int stride)
ccl_device void film_adaptive_sampling_filter_x(KernelGlobals kg, ccl_global float *render_buffer, const int y, const int start_x, const int width, const int offset, const int stride)
ccl_device bool film_adaptive_sampling_convergence_check(KernelGlobals kg, ccl_global float *render_buffer, const int x, const int y, const float threshold, const int reset, const int offset, const int stride)
void KERNEL_FUNCTION_FULL_NAME volume_guiding_filter_x(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, const int y, const int center_x, const int min_x, const int max_x, const int offset, int stride)
void KERNEL_FUNCTION_FULL_NAME volume_guiding_filter_y(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, const int x, const int center_y, const int height, const int offset, int stride)
void KERNEL_FUNCTION_FULL_NAME shader_eval_background(const ThreadKernelGlobalsCPU *kg, const KernelShaderEvalInput *input, float *output, const int offset)
bool KERNEL_FUNCTION_FULL_NAME adaptive_sampling_convergence_check(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, const int x, const int y, const float threshold, const int reset, const int offset, int stride)
void KERNEL_FUNCTION_FULL_NAME cryptomatte_postprocess(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, int pixel_index)
void KERNEL_FUNCTION_FULL_NAME shader_eval_displace(const ThreadKernelGlobalsCPU *kg, const KernelShaderEvalInput *input, float *output, const int offset)
void KERNEL_FUNCTION_FULL_NAME adaptive_sampling_filter_x(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, const int y, const int start_x, const int width, const int offset, int stride)
void KERNEL_FUNCTION_FULL_NAME adaptive_sampling_filter_y(const ThreadKernelGlobalsCPU *kg, ccl_global float *render_buffer, const int x, const int start_y, const int height, const int offset, int stride)
void KERNEL_FUNCTION_FULL_NAME shader_eval_curve_shadow_transparency(const ThreadKernelGlobalsCPU *kg, const KernelShaderEvalInput *input, float *output, const int offset)
void KERNEL_FUNCTION_FULL_NAME shader_eval_volume_density(const ThreadKernelGlobalsCPU *kg, const KernelShaderEvalInput *input, float *output, const int offset)
ccl_device_inline uint popcount(const uint x)
Definition math_base.h:734
std::shared_ptr< const T > get(const GenericKey &key, FunctionRef< std::unique_ptr< T >()> compute_fn)
unsigned int uint
Definition BKE_text.h:55
unsigned short ushort
Definition BKE_text.h:56
#define gpu_parallel_active_index_array
CCL_NAMESPACE_BEGIN __device__ void gpu_parallel_prefix_sum(const int global_id, ccl_global int *ccl_restrict counter, ccl_global int *ccl_restrict prefix_sum, const int num_values)
#define gpu_parallel_sort_write_pass
__device__ void gpu_parallel_sorted_index_array(const uint state_index, const uint num_states, const int num_states_limit, ccl_global int *indices, ccl_global int *num_indices, ccl_global int *key_counter, ccl_global int *key_prefix_sum, GetKeyOp get_key_op)
#define gpu_parallel_sort_bucket_pass
ccl_device void integrator_shade_background(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
CCL_NAMESPACE_BEGIN ccl_device void integrator_shade_dedicated_light(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict)
ccl_device void integrator_shade_light_nee(KernelGlobals kg, IntegratorShadowState state, ccl_global float *ccl_restrict)
ccl_device void integrator_shade_light_forward(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
Definition shade_light.h:79
ccl_device void integrator_shade_shadow(KernelGlobals kg, IntegratorShadowState state, ccl_global float *ccl_restrict render_buffer)
ccl_device_forceinline void integrator_shade_surface(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
ccl_device_forceinline void integrator_shade_surface_raytrace(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
ccl_device void integrator_shade_volume_ray_marching(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
ccl_device void integrator_shade_volume(KernelGlobals kg, IntegratorState state, ccl_global float *ccl_restrict render_buffer)
ccl_device_inline bool kernel_shadow_catcher_path_can_split(ConstIntegratorState state)
#define INTEGRATOR_STATE_WRITE(state, nested_struct, member)
Definition state.h:235
#define INTEGRATOR_STATE(state, nested_struct, member)
Definition state.h:234
float x
float y
float y
Definition sky_math.h:225
float z
Definition sky_math.h:225
float x
Definition sky_math.h:225
float w
Definition sky_math.h:225
Definition half.h:53
half x
Definition half.h:54
half w
Definition half.h:54
half z
Definition half.h:54
half y
Definition half.h:54
CCL_NAMESPACE_BEGIN ccl_device_inline void get_work_pixel(const ccl_global KernelWorkTile *tile, const uint global_work_index, ccl_private uint *x, ccl_private uint *y, ccl_private uint *sample)