8#define __KERNEL_METAL__
9#define CCL_NAMESPACE_BEGIN
10#define CCL_NAMESPACE_END
12#ifndef ATTR_FALLTHROUGH
13# define ATTR_FALLTHROUGH
16#include <metal_atomic>
18#include <metal_stdlib>
21#pragma clang diagnostic ignored "-Wunused-variable"
22#pragma clang diagnostic ignored "-Wsign-compare"
23#pragma clang diagnostic ignored "-Wuninitialized"
24#pragma clang diagnostic ignored "-Wc++17-extensions"
25#pragma clang diagnostic ignored "-Wmacro-redefined"
30#define ccl_device_inline ccl_device __attribute__((always_inline))
31#define ccl_device_forceinline ccl_device __attribute__((always_inline))
32#if defined(__KERNEL_METAL_APPLE__)
33# define ccl_device_noinline ccl_device
35# define ccl_device_noinline ccl_device __attribute__((noinline))
38#define ccl_device_extern extern "C"
39#define ccl_device_noinline_cpu ccl_device
40#define ccl_device_inline_method ccl_device
41#define ccl_device_template_spec template<> ccl_device_inline
42#define ccl_global device
43#define ccl_inline_constant static constant constexpr
44#define ccl_device_constant constant
45#define ccl_static_constexpr static constant constexpr
46#define ccl_constant constant
47#define ccl_gpu_shared threadgroup
48#define ccl_private thread
49#ifdef __KERNEL_METALRT__
50# define ccl_ray_data ray_data
52# define ccl_ray_data ccl_private
55#define ccl_restrict __restrict
56#define ccl_align(n) alignas(n)
57#define ccl_optional_struct_init
58#define ccl_attr_maybe_unused
64#define kernel_assert(cond)
66#define offsetof(t, d) __builtin_offsetof(t, d)
68#define ccl_gpu_global_id_x() metal_global_id
69#define ccl_gpu_warp_size simdgroup_size
70#define ccl_gpu_thread_idx_x simd_group_index
71#define ccl_gpu_thread_mask(thread_warp) uint64_t((1ull << thread_warp) - 1)
73#define ccl_gpu_ballot(predicate) \
74 ((uint64_t)((metal::simd_vote::vote_t)metal::simd_ballot(predicate)))
75#define ccl_gpu_syncthreads() threadgroup_barrier(metal::mem_flags::mem_threadgroup);
81#define ccl_gpu_kernel(block_num_threads, thread_num_registers)
82#define ccl_gpu_kernel_threads(block_num_threads)
88#define FN2(p1, p2) p1; p2;
89#define FN3(p1, p2, p3) p1; p2; p3;
90#define FN4(p1, p2, p3, p4) p1; p2; p3; p4;
91#define FN5(p1, p2, p3, p4, p5) p1; p2; p3; p4; p5;
92#define FN6(p1, p2, p3, p4, p5, p6) p1; p2; p3; p4; p5; p6;
93#define FN7(p1, p2, p3, p4, p5, p6, p7) p1; p2; p3; p4; p5; p6; p7;
94#define FN8(p1, p2, p3, p4, p5, p6, p7, p8) p1; p2; p3; p4; p5; p6; p7; p8;
95#define FN9(p1, p2, p3, p4, p5, p6, p7, p8, p9) p1; p2; p3; p4; p5; p6; p7; p8; p9;
96#define FN10(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10;
97#define FN11(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11;
98#define FN12(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12;
99#define FN13(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13;
100#define FN14(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14;
101#define FN15(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15;
102#define FN16(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16;
103#define FN17(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17;
104#define FN18(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18;
105#define FN19(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19;
106#define FN20(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19; p20;
107#define FN21(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19; p20; p21;
108#define FN22(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19; p20; p21; p22;
109#define FN23(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22, p23) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19; p20; p21; p22; p23;
110#define FN24(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22, p23, p24) p1; p2; p3; p4; p5; p6; p7; p8; p9; p10; p11; p12; p13; p14; p15; p16; p17; p18; p19; p20; p21; p22; p23; p24;
111#define GET_LAST_ARG(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, p22, p23, p24, ...) p24
112#define PARAMS_MAKER(...) GET_LAST_ARG(__VA_ARGS__, FN24, FN23, FN22, FN21, FN20, FN19, FN18, FN17, FN16, FN15, FN14, FN13, FN12, FN11, FN10, FN9, FN8, FN7, FN6, FN5, FN4, FN3, FN2, FN1, FN0)
117#ifdef __METAL_GLOBAL_BUILTINS__
119#define ccl_gpu_kernel_signature(name, ...) \
120struct kernel_gpu_## name \
122 PARAMS_MAKER(__VA_ARGS__)(__VA_ARGS__) \
123 void run(thread MetalKernelContext& context, \
124 threadgroup metal::atomic_int *threadgroup_array) ccl_global const; \
126kernel void cycles_metal_## name(device const kernel_gpu_## name *params_struct, \
127 constant KernelParamsMetal &ccl_restrict _launch_params_metal, \
128 constant MetalAncillaries *_metal_ancillaries, \
129 threadgroup metal::atomic_int *threadgroup_array[[ threadgroup(0) ]]) { \
130 MetalKernelContext context(_launch_params_metal, _metal_ancillaries); \
131 params_struct->run(context, threadgroup_array); \
133void kernel_gpu_## name::run(thread MetalKernelContext& context, \
134 threadgroup metal::atomic_int *threadgroup_array) ccl_global const
141#define ccl_gpu_kernel_signature(name, ...) \
142struct kernel_gpu_## name \
144 PARAMS_MAKER(__VA_ARGS__)(__VA_ARGS__) \
145 void run(thread MetalKernelContext& context, \
146 threadgroup metal::atomic_int *threadgroup_array, \
147 const uint metal_global_id, \
148 const ushort metal_local_id, \
149 const ushort metal_local_size, \
150 const uint metal_grid_id, \
151 uint simdgroup_size, \
152 uint simd_lane_index, \
153 uint simd_group_index, \
154 uint num_simd_groups) ccl_global const; \
156kernel void cycles_metal_## name(device const kernel_gpu_## name *params_struct, \
157 constant KernelParamsMetal &ccl_restrict _launch_params_metal, \
158 constant MetalAncillaries *_metal_ancillaries, \
159 threadgroup metal::atomic_int *threadgroup_array[[ threadgroup(0) ]], \
160 const uint metal_global_id [[thread_position_in_grid]], \
161 const ushort metal_local_id [[thread_position_in_threadgroup]], \
162 const ushort metal_local_size [[threads_per_threadgroup]], \
163 const uint metal_grid_id [[threadgroup_position_in_grid]], \
164 uint simdgroup_size [[threads_per_simdgroup]], \
165 uint simd_lane_index [[thread_index_in_simdgroup]], \
166 uint simd_group_index [[simdgroup_index_in_threadgroup]], \
167 uint num_simd_groups [[simdgroups_per_threadgroup]]) { \
168 MetalKernelContext context(_launch_params_metal, _metal_ancillaries); \
169 params_struct->run(context, threadgroup_array, metal_global_id, metal_local_id, metal_local_size, metal_grid_id, simdgroup_size, simd_lane_index, simd_group_index, num_simd_groups); \
171void kernel_gpu_## name::run(thread MetalKernelContext& context, \
172 threadgroup metal::atomic_int *threadgroup_array, \
173 const uint metal_global_id, \
174 const ushort metal_local_id, \
175 const ushort metal_local_size, \
176 const uint metal_grid_id, \
177 uint simdgroup_size, \
178 uint simd_lane_index, \
179 uint simd_group_index, \
180 uint num_simd_groups) ccl_global const
184#define ccl_gpu_kernel_postfix
185#define ccl_gpu_kernel_call(x) context.x
186#define ccl_gpu_kernel_within_bounds(i,n) true
189#define ccl_gpu_kernel_lambda(func, ...) \
190 struct KernelLambda \
192 KernelLambda(ccl_private MetalKernelContext &_context) : context(_context) {} \
193 ccl_private MetalKernelContext &context; \
195 int operator()(const int state) const { return (func); } \
196 } ccl_gpu_kernel_lambda_pass(context)
259#define __uint_as_float(x) as_type<float>(x)
260#define __float_as_uint(x) as_type<uint>(x)
261#define __int_as_float(x) as_type<float>(x)
262#define __float_as_int(x) as_type<int>(x)
263#define __float2half(x) half(x)
264#define powf(x, y) metal::pow(float(x), float(y))
265#define fabsf(x) metal::fabs(float(x))
266#define copysignf(x, y) metal::copysign(float(x), float(y))
267#define asinf(x) metal::asin(float(x))
268#define acosf(x) metal::acos(float(x))
269#define atanf(x) metal::atan(float(x))
270#define floorf(x) metal::floor(float(x))
271#define ceilf(x) metal::ceil(float(x))
272#define roundf(x) metal::round(float(x))
273#define hypotf(x, y) metal::hypot(float(x), float(y))
274#define atan2f(x, y) metal::atan2(float(x), float(y))
275#define fmaxf(x, y) metal::fmax(float(x), float(y))
276#define fminf(x, y) metal::fmin(float(x), float(y))
277#define fmodf(x, y) metal::fmod(float(x), float(y))
278#define sinhf(x) metal::sinh(float(x))
279#define coshf(x) metal::cosh(float(x))
280#define tanhf(x) metal::tanh(float(x))
281#define saturatef(x) metal::saturate(float(x))
282#define ldexpf(x, y) metal::ldexp(float(x), int(y))
286#define trigmode metal::fast
287#define sinf(x) trigmode::sin(float(x))
288#define cosf(x) trigmode::cos(float(x))
289#define tanf(x) trigmode::tan(float(x))
290#define expf(x) trigmode::exp(float(x))
291#define sqrtf(x) trigmode::sqrt(float(x))
292#define logf(x) trigmode::log(float(x))
300using metal::distance;
304using metal::faceforward;
311using metal::normalize;
315using metal::saturate;
323#ifdef __KERNEL_METALRT__
325# if defined(__METALRT_MOTION__)
326# define METALRT_TAGS \
327 metal::raytracing::instancing, metal::raytracing::instance_motion, \
328 metal::raytracing::primitive_motion
329# define METALRT_BLAS_TAGS , metal::raytracing::primitive_motion
331# define METALRT_TAGS metal::raytracing::instancing
332# define METALRT_BLAS_TAGS
335# if defined(__METALRT_EXTENDED_LIMITS__)
336# define METALRT_LIMITS , metal::raytracing::extended_limits
338# define METALRT_LIMITS
341typedef metal::raytracing::acceleration_structure<METALRT_TAGS> metalrt_as_type;
342typedef metal::raytracing::intersection_function_table<metal::raytracing::triangle_data,
343 metal::raytracing::curve_data,
344 METALRT_TAGS METALRT_LIMITS>
346typedef metal::raytracing::intersector<metal::raytracing::triangle_data,
347 metal::raytracing::curve_data,
348 METALRT_TAGS METALRT_LIMITS>
349 metalrt_intersector_type;
350# if defined(__METALRT_MOTION__)
351typedef metal::raytracing::acceleration_structure<metal::raytracing::primitive_motion>
352 metalrt_blas_as_type;
353typedef metal::raytracing::intersection_function_table<
354 metal::raytracing::triangle_data,
355 metal::raytracing::curve_data,
356 metal::raytracing::primitive_motion METALRT_LIMITS>
357 metalrt_blas_ift_type;
358typedef metal::raytracing::intersector<metal::raytracing::triangle_data,
359 metal::raytracing::curve_data,
360 metal::raytracing::primitive_motion METALRT_LIMITS>
361 metalrt_blas_intersector_type;
363typedef metal::raytracing::acceleration_structure<> metalrt_blas_as_type;
364typedef metal::raytracing::intersection_function_table<
365 metal::raytracing::triangle_data,
366 metal::raytracing::curve_data METALRT_LIMITS>
367 metalrt_blas_ift_type;
368typedef metal::raytracing::intersector<metal::raytracing::triangle_data,
369 metal::raytracing::curve_data METALRT_LIMITS>
370 metalrt_blas_intersector_type;
382 metal::texture2d<float, metal::access::sample>
tex;
385#ifdef __KERNEL_METALRT__
386struct MetalRTBlasWrapper {
387 metalrt_blas_as_type blas;
397#ifdef __KERNEL_METALRT__
398 metalrt_as_type accel_struct;
399 constant MetalRTBlasWrapper *blas_accel_structs;
400 metalrt_ift_type ift_default;
401 metalrt_ift_type ift_shadow;
402 metalrt_ift_type ift_shadow_all;
403 metalrt_ift_type ift_volume;
404 metalrt_blas_ift_type ift_local;
405 metalrt_ift_type ift_local_mblur;
406 metalrt_blas_ift_type ift_local_single_hit;
407 metalrt_ift_type ift_local_single_hit_mblur;
429 metal::sampler(metal::address::repeat, metal::filter::nearest),
430 metal::sampler(metal::address::clamp_to_edge, metal::filter::nearest),
431 metal::sampler(metal::address::clamp_to_zero, metal::filter::nearest),
432 metal::sampler(metal::address::mirrored_repeat, metal::filter::nearest),
433 metal::sampler(metal::address::repeat, metal::filter::linear),
434 metal::sampler(metal::address::clamp_to_edge, metal::filter::linear),
435 metal::sampler(metal::address::clamp_to_zero, metal::filter::linear),
436 metal::sampler(metal::address::mirrored_repeat, metal::filter::linear),
439#ifdef __METAL_GLOBAL_BUILTINS__
440const uint metal_global_id [[thread_position_in_grid]];
441const ushort metal_local_id [[thread_position_in_threadgroup]];
442const ushort metal_local_size [[threads_per_threadgroup]];
443const uint metal_grid_id [[threadgroup_position_in_grid]];
444const uint simdgroup_size [[threads_per_simdgroup]];
445const uint simd_lane_index [[thread_index_in_simdgroup]];
446const uint simd_group_index [[simdgroup_index_in_threadgroup]];
447const uint num_simd_groups [[simdgroups_per_threadgroup]];
unsigned long long int uint64_t
SIMD_FORCE_INLINE const btScalar & z() const
Return the z value.
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
#define ccl_device_forceinline
metal::texture2d< float, metal::access::sample > tex