20#if defined(FREE_WINDOWS64)
22#elif defined(_MSC_VER) && !defined(__KERNEL_NEON__)
24#elif (defined(__x86_64__) || defined(__i386__))
25# include <x86intrin.h>
26#elif defined(__KERNEL_NEON__)
27# define SSE2NEON_PRECISE_MINMAX 1
32#if defined(__x86_64__) || defined(_M_X64)
33# define SIMD_SET_FLUSH_TO_ZERO \
34 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
35 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
36#elif defined(__aarch64__) || defined(_M_ARM64)
39# if defined(_MM_SET_FLUSH_ZERO_MODE)
40# define SIMD_SET_FLUSH_TO_ZERO \
41 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
42 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
43# elif !defined(_M_ARM64)
44# define _MM_FLUSH_ZERO_ON 24
45# define __get_fpcr(__fpcr) __asm__ __volatile__("mrs %0,fpcr" : "=r"(__fpcr))
46# define __set_fpcr(__fpcr) __asm__ __volatile__("msr fpcr,%0" : : "ri"(__fpcr))
47# define SIMD_SET_FLUSH_TO_ZERO set_fz(_MM_FLUSH_ZERO_ON);
48# define SIMD_GET_FLUSH_TO_ZERO get_fz(_MM_FLUSH_ZERO_ON)
50# define _MM_FLUSH_ZERO_ON 24
51# define __get_fpcr(__fpcr) __fpcr = _ReadStatusReg(0x5A20)
52# define __set_fpcr(__fpcr) _WriteStatusReg(0x5A20, __fpcr)
53# define SIMD_SET_FLUSH_TO_ZERO set_fz(_MM_FLUSH_ZERO_ON);
54# define SIMD_GET_FLUSH_TO_ZERO get_fz(_MM_FLUSH_ZERO_ON)
57# define SIMD_SET_FLUSH_TO_ZERO
70#if defined(__x86_64__) || defined(_M_X64) || \
71 ((defined(__aarch64__) || defined(_M_ARM64)) && defined(_MM_SET_FLUSH_ZERO_MODE))
80 _mm_setcsr(orig_csr_);
84 unsigned int orig_csr_;
91extern const __m128 _mm_lookupmask_ps[16];
100static struct FalseTy {
107static struct ZeroTy {
129static struct NegInfTy {
132 return -std::numeric_limits<float>::infinity();
136 return std::numeric_limits<int>::min();
140static struct PosInfTy {
143 return std::numeric_limits<float>::infinity();
147 return std::numeric_limits<int>::max();
151static struct StepTy {
155#if (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_MM_SET_FLUSH_ZERO_MODE)
160 __get_fpcr(old_fpcr);
161 new_fpcr = old_fpcr | (
uint64_t(1) << flag);
162 __set_fpcr(new_fpcr);
163 __get_fpcr(old_fpcr);
164 return old_fpcr == new_fpcr;
169 __get_fpcr(cur_fpcr);
170 return (cur_fpcr & (
uint64_t(1) << flag)) > 0 ? 1 : 0;
175#if defined(__KERNEL_NEON__)
176template<
class type, const
int i0, const
int i1, const
int i2, const
int i3>
177type shuffle_neon(
const type &a)
179 if (i0 == i1 && i0 == i2 && i0 == i3) {
180 return type(vdupq_laneq_s32(int32x4_t(a), i0));
182 static const uint8_t tbl[16] = {(i0 * 4) + 0,
199 return type(vqtbl1q_s8(int8x16_t(a), *(uint8x16_t *)tbl));
202template<
class type, const
int i0, const
int i1, const
int i2, const
int i3>
203type shuffle_neon(
const type &a,
const type &
b)
206 static const uint8_t tbl[16] = {(i0 * 4) + 0,
223 return type(vqtbl1q_s8(int8x16_t(
b), *(uint8x16_t *)tbl));
227 static const uint8_t tbl[16] = {(i0 * 4) + 0,
246 int8x16x2_t t = {int8x16_t(a), int8x16_t(
b)};
247 uint8x16_t idx = *(uint8x16_t *)tbl;
248 return type(vqtbl2q_s8(t, idx));
257#if defined(__BMI__) && defined(__GNUC__)
259# define _tzcnt_u32 __tzcnt_u32
262# define _tzcnt_u64 __tzcnt_u64
266#if defined(__LZCNT__)
267# define _lzcnt_u32 __lzcnt32
268# define _lzcnt_u64 __lzcnt64
271#if defined(_WIN32) && !defined(__MINGW32__) && !defined(__clang__)
275# if defined(__KERNEL_AVX2__)
276 return _tzcnt_u32(v);
279 _BitScanForward(&r, v);
287 _BitScanReverse(&r, v);
294 _bittestandcomplement(&r, i);
300# if defined(__KERNEL_AVX2__)
301 return _tzcnt_u32(v);
307# if defined(__KERNEL_64_BIT__)
311# if defined(__KERNEL_AVX2__)
312 return _tzcnt_u64(v);
315 _BitScanForward64(&r, v);
323 _BitScanReverse64(&r, v);
330 _bittestandcomplement64((__int64 *)&r, i);
336# if defined(__KERNEL_AVX2__)
337# if defined(__KERNEL_64_BIT__)
338 return _tzcnt_u64(v);
340 return _tzcnt_u32(v);
349#elif (defined(__x86_64__) || defined(__i386__)) && defined(__KERNEL_SSE2__)
355 asm(
"bsf %1,%0" :
"=r"(r) :
"r"(
v));
362 asm(
"bsr %1,%0" :
"=r"(r) :
"r"(
v));
369 asm(
"btc %1,%0" :
"=r"(r) :
"r"(i),
"0"(
v) :
"flags");
373# if (defined(__KERNEL_64_BIT__) || defined(__APPLE__)) && \
374 !(defined(__ILP32__) && defined(__x86_64__))
378 asm(
"bsf %1,%0" :
"=r"(r) :
"r"(
v));
386 asm(
"bsr %1,%0" :
"=r"(r) :
"r"(
v));
393 asm(
"btc %1,%0" :
"=r"(r) :
"r"(i),
"0"(
v) :
"flags");
399# if defined(__KERNEL_AVX2__)
400 return _tzcnt_u32(v);
406# if (defined(__KERNEL_64_BIT__) || defined(__APPLE__)) && \
407 !(defined(__ILP32__) && defined(__x86_64__))
410# if defined(__KERNEL_AVX2__)
411# if defined(__KERNEL_64_BIT__)
412 return _tzcnt_u64(v);
414 return _tzcnt_u32(v);
426 for (uint32_t i = 0; i < 32; i++) {
427 if (
x & (uint32_t(1) << i)) {
436 for (uint32_t i = 0; i < 32; i++) {
437 if (
x & (uint32_t(1) << (31 - i))) {
446 const uint32_t
mask = uint32_t(1) << bit;
452 for (uint32_t i = 0; i < 64; i++) {
462 for (uint32_t i = 0; i < 64; i++) {
492#if defined(__KERNEL_AVX__) || defined(__KERNEL_AVX2__)
493# undef _mm256_cvtss_f32
494# define _mm256_cvtss_f32(a) (_mm_cvtss_f32(_mm256_castps256_ps128(a)))
498#if defined(__KERNEL_SSE2__) || defined(__KERNEL_SSE3__) || defined(__KERNEL_SSSE3__) || \
499 defined(__KERNEL_SSE42__) || defined(__KERNEL_AVX__) || defined(__KERNEL_AVX2__)
unsigned long long int uint64_t
#define ccl_attr_maybe_unused
#define CCL_NAMESPACE_END
#define assert(assertion)
VecBase< float, D > step(VecOp< float, D >, VecOp< float, D >) RET
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
ATTR_WARN_UNUSED_RESULT const BMVert * v
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
#define SIMD_SET_FLUSH_TO_ZERO
__forceinline uint32_t bitscan(const uint32_t value)
__forceinline uint32_t __bsr(const uint32_t x)
__forceinline uint32_t __btc(const uint32_t x, const uint32_t bit)
__forceinline uint32_t __bsf(const uint32_t x)