Blender V5.3
simd.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2013 Intel Corporation
2 * SPDX-FileCopyrightText: 2014-2022 Blender Foundation
3 *
4 * SPDX-License-Identifier: Apache-2.0 */
5
6#pragma once
7
8#include <cstdint>
9#include <limits>
10
11#include "util/defines.h"
12#include "util/optimization.h"
13
14/* SSE Intrinsics includes
15 *
16 * We assume __KERNEL_SSEX__ flags to have been defined at this point.
17 *
18 * MinGW64 has conflicting declarations for these SSE headers in <windows.h>.
19 * Since we can't avoid including <windows.h>, better only include that */
20#if defined(FREE_WINDOWS64)
21# include "util/windows.h"
22#elif defined(_MSC_VER) && !defined(__KERNEL_NEON__)
23# include <intrin.h>
24#elif (defined(__x86_64__) || defined(__i386__))
25# include <x86intrin.h>
26#elif defined(__KERNEL_NEON__)
27# define SSE2NEON_PRECISE_MINMAX 1
28# include <sse2neon.h>
29#endif
30
31/* Floating Point Control, for Embree. */
32#if defined(__x86_64__) || defined(_M_X64)
33# define SIMD_SET_FLUSH_TO_ZERO \
34 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
35 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
36#elif defined(__aarch64__) || defined(_M_ARM64)
37/* The get/set denormals to zero was implemented in sse2neon v1.5.0.
38 * Keep the compatibility code until the minimum library version is increased. */
39# if defined(_MM_SET_FLUSH_ZERO_MODE)
40# define SIMD_SET_FLUSH_TO_ZERO \
41 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
42 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
43# elif !defined(_M_ARM64)
44# define _MM_FLUSH_ZERO_ON 24
45# define __get_fpcr(__fpcr) __asm__ __volatile__("mrs %0,fpcr" : "=r"(__fpcr))
46# define __set_fpcr(__fpcr) __asm__ __volatile__("msr fpcr,%0" : : "ri"(__fpcr))
47# define SIMD_SET_FLUSH_TO_ZERO set_fz(_MM_FLUSH_ZERO_ON);
48# define SIMD_GET_FLUSH_TO_ZERO get_fz(_MM_FLUSH_ZERO_ON)
49# else
50# define _MM_FLUSH_ZERO_ON 24
51# define __get_fpcr(__fpcr) __fpcr = _ReadStatusReg(0x5A20)
52# define __set_fpcr(__fpcr) _WriteStatusReg(0x5A20, __fpcr)
53# define SIMD_SET_FLUSH_TO_ZERO set_fz(_MM_FLUSH_ZERO_ON);
54# define SIMD_GET_FLUSH_TO_ZERO get_fz(_MM_FLUSH_ZERO_ON)
55# endif
56#else
57# define SIMD_SET_FLUSH_TO_ZERO
58#endif
59
61
62/* Enable flush-to-zero and denormals-are-zero for the lifetime of the object, then restore the
63 * previous state. This is the mode threads that use Embree run in, so this is mainly useful to
64 * check that code behaves the same in both modes.
65 *
66 * Does nothing on platforms where the control register is not accessible this way. Those use the
67 * `set_fz` path above, which can not read back the original state. */
69/* Same platforms that use the `_mm_*` intrinsics for SIMD_SET_FLUSH_TO_ZERO above. */
70#if defined(__x86_64__) || defined(_M_X64) || \
71 ((defined(__aarch64__) || defined(_M_ARM64)) && defined(_MM_SET_FLUSH_ZERO_MODE))
72 public:
73 ScopedFlushToZero() : orig_csr_(_mm_getcsr())
74 {
76 }
77
79 {
80 _mm_setcsr(orig_csr_);
81 }
82
83 private:
84 unsigned int orig_csr_;
85#endif
86};
87
88/* Data structures used by SSE classes. */
89#ifdef __KERNEL_SSE2__
90
91extern const __m128 _mm_lookupmask_ps[16];
92
93static struct TrueTy {
94 __forceinline operator bool() const
95 {
96 return true;
97 }
99
100static struct FalseTy {
101 __forceinline operator bool() const
102 {
103 return false;
104 }
106
107static struct ZeroTy {
108 __forceinline operator float() const
109 {
110 return 0;
111 }
112 __forceinline operator int() const
113 {
114 return 0;
115 }
117
118static struct OneTy {
119 __forceinline operator float() const
120 {
121 return 1;
122 }
123 __forceinline operator int() const
124 {
125 return 1;
126 }
128
129static struct NegInfTy {
130 __forceinline operator float() const
131 {
132 return -std::numeric_limits<float>::infinity();
133 }
134 __forceinline operator int() const
135 {
136 return std::numeric_limits<int>::min();
137 }
138} neg_inf ccl_attr_maybe_unused;
139
140static struct PosInfTy {
141 __forceinline operator float() const
142 {
143 return std::numeric_limits<float>::infinity();
144 }
145 __forceinline operator int() const
146 {
147 return std::numeric_limits<int>::max();
148 }
150
151static struct StepTy {
153
154#endif
155#if (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_MM_SET_FLUSH_ZERO_MODE)
156__forceinline int set_fz(const uint32_t flag)
157{
158 uint64_t old_fpcr;
159 uint64_t new_fpcr;
160 __get_fpcr(old_fpcr);
161 new_fpcr = old_fpcr | (uint64_t(1) << flag);
162 __set_fpcr(new_fpcr);
163 __get_fpcr(old_fpcr);
164 return old_fpcr == new_fpcr;
165}
166__forceinline int get_fz(const uint32_t flag)
167{
168 uint64_t cur_fpcr;
169 __get_fpcr(cur_fpcr);
170 return (cur_fpcr & (uint64_t(1) << flag)) > 0 ? 1 : 0;
171}
172#endif
173
174/* Utilities used by Neon */
175#if defined(__KERNEL_NEON__)
176template<class type, const int i0, const int i1, const int i2, const int i3>
177type shuffle_neon(const type &a)
178{
179 if (i0 == i1 && i0 == i2 && i0 == i3) {
180 return type(vdupq_laneq_s32(int32x4_t(a), i0));
181 }
182 static const uint8_t tbl[16] = {(i0 * 4) + 0,
183 (i0 * 4) + 1,
184 (i0 * 4) + 2,
185 (i0 * 4) + 3,
186 (i1 * 4) + 0,
187 (i1 * 4) + 1,
188 (i1 * 4) + 2,
189 (i1 * 4) + 3,
190 (i2 * 4) + 0,
191 (i2 * 4) + 1,
192 (i2 * 4) + 2,
193 (i2 * 4) + 3,
194 (i3 * 4) + 0,
195 (i3 * 4) + 1,
196 (i3 * 4) + 2,
197 (i3 * 4) + 3};
198
199 return type(vqtbl1q_s8(int8x16_t(a), *(uint8x16_t *)tbl));
200}
201
202template<class type, const int i0, const int i1, const int i2, const int i3>
203type shuffle_neon(const type &a, const type &b)
204{
205 if (&a == &b) {
206 static const uint8_t tbl[16] = {(i0 * 4) + 0,
207 (i0 * 4) + 1,
208 (i0 * 4) + 2,
209 (i0 * 4) + 3,
210 (i1 * 4) + 0,
211 (i1 * 4) + 1,
212 (i1 * 4) + 2,
213 (i1 * 4) + 3,
214 (i2 * 4) + 0,
215 (i2 * 4) + 1,
216 (i2 * 4) + 2,
217 (i2 * 4) + 3,
218 (i3 * 4) + 0,
219 (i3 * 4) + 1,
220 (i3 * 4) + 2,
221 (i3 * 4) + 3};
222
223 return type(vqtbl1q_s8(int8x16_t(b), *(uint8x16_t *)tbl));
224 }
225 else {
226
227 static const uint8_t tbl[16] = {(i0 * 4) + 0,
228 (i0 * 4) + 1,
229 (i0 * 4) + 2,
230 (i0 * 4) + 3,
231 (i1 * 4) + 0,
232 (i1 * 4) + 1,
233 (i1 * 4) + 2,
234 (i1 * 4) + 3,
235 (i2 * 4) + 0 + 16,
236 (i2 * 4) + 1 + 16,
237 (i2 * 4) + 2 + 16,
238 (i2 * 4) + 3 + 16,
239 (i3 * 4) + 0 + 16,
240 (i3 * 4) + 1 + 16,
241 (i3 * 4) + 2 + 16,
242 (i3 * 4) + 3 + 16};
243
244 /* NOTE: This cannot all be put in a single line due to how MSVC ARM64
245 * implements the function calls as several layers of macros. */
246 int8x16x2_t t = {int8x16_t(a), int8x16_t(b)};
247 uint8x16_t idx = *(uint8x16_t *)tbl;
248 return type(vqtbl2q_s8(t, idx));
249 }
250}
251#endif /* __KERNEL_NEON */
252
253/* Intrinsics Functions
254 *
255 * For fast bit operations. */
256
257#if defined(__BMI__) && defined(__GNUC__)
258# ifndef _tzcnt_u32
259# define _tzcnt_u32 __tzcnt_u32
260# endif
261# ifndef _tzcnt_u64
262# define _tzcnt_u64 __tzcnt_u64
263# endif
264#endif
265
266#if defined(__LZCNT__)
267# define _lzcnt_u32 __lzcnt32
268# define _lzcnt_u64 __lzcnt64
269#endif
270
271#if defined(_WIN32) && !defined(__MINGW32__) && !defined(__clang__)
272/* Intrinsic functions on Windows. */
273__forceinline uint32_t __bsf(const uint32_t v)
274{
275# if defined(__KERNEL_AVX2__)
276 return _tzcnt_u32(v);
277# else
278 unsigned long r = 0;
279 _BitScanForward(&r, v);
280 return r;
281# endif
282}
283
284__forceinline uint32_t __bsr(const uint32_t v)
285{
286 unsigned long r = 0;
287 _BitScanReverse(&r, v);
288 return r;
289}
290
291__forceinline uint32_t __btc(const uint32_t v, const uint32_t i)
292{
293 long r = v;
294 _bittestandcomplement(&r, i);
295 return r;
296}
297
298__forceinline uint32_t bitscan(const uint32_t v)
299{
300# if defined(__KERNEL_AVX2__)
301 return _tzcnt_u32(v);
302# else
303 return __bsf(v);
304# endif
305}
306
307# if defined(__KERNEL_64_BIT__)
308
310{
311# if defined(__KERNEL_AVX2__)
312 return _tzcnt_u64(v);
313# else
314 unsigned long r = 0;
315 _BitScanForward64(&r, v);
316 return r;
317# endif
318}
319
321{
322 unsigned long r = 0;
323 _BitScanReverse64(&r, v);
324 return r;
325}
326
328{
329 uint64_t r = v;
330 _bittestandcomplement64((__int64 *)&r, i);
331 return r;
332}
333
335{
336# if defined(__KERNEL_AVX2__)
337# if defined(__KERNEL_64_BIT__)
338 return _tzcnt_u64(v);
339# else
340 return _tzcnt_u32(v);
341# endif
342# else
343 return __bsf(v);
344# endif
345}
346
347# endif /* __KERNEL_64_BIT__ */
348
349#elif (defined(__x86_64__) || defined(__i386__)) && defined(__KERNEL_SSE2__)
350/* Intrinsic functions with x86 SSE. */
351
352__forceinline uint32_t __bsf(const uint32_t v)
353{
354 uint32_t r = 0;
355 asm("bsf %1,%0" : "=r"(r) : "r"(v));
356 return r;
357}
358
359__forceinline uint32_t __bsr(const uint32_t v)
360{
361 uint32_t r = 0;
362 asm("bsr %1,%0" : "=r"(r) : "r"(v));
363 return r;
364}
365
366__forceinline uint32_t __btc(const uint32_t v, const uint32_t i)
367{
368 uint32_t r = 0;
369 asm("btc %1,%0" : "=r"(r) : "r"(i), "0"(v) : "flags");
370 return r;
371}
372
373# if (defined(__KERNEL_64_BIT__) || defined(__APPLE__)) && \
374 !(defined(__ILP32__) && defined(__x86_64__))
376{
377 uint64_t r = 0;
378 asm("bsf %1,%0" : "=r"(r) : "r"(v));
379 return r;
380}
381# endif
382
384{
385 uint64_t r = 0;
386 asm("bsr %1,%0" : "=r"(r) : "r"(v));
387 return r;
388}
389
391{
392 uint64_t r = 0;
393 asm("btc %1,%0" : "=r"(r) : "r"(i), "0"(v) : "flags");
394 return r;
395}
396
397__forceinline uint32_t bitscan(const uint32_t v)
398{
399# if defined(__KERNEL_AVX2__)
400 return _tzcnt_u32(v);
401# else
402 return __bsf(v);
403# endif
404}
405
406# if (defined(__KERNEL_64_BIT__) || defined(__APPLE__)) && \
407 !(defined(__ILP32__) && defined(__x86_64__))
409{
410# if defined(__KERNEL_AVX2__)
411# if defined(__KERNEL_64_BIT__)
412 return _tzcnt_u64(v);
413# else
414 return _tzcnt_u32(v);
415# endif
416# else
417 return __bsf(v);
418# endif
419}
420# endif
421
422#else
423/* Intrinsic functions fallback for arbitrary processor. */
424__forceinline uint32_t __bsf(const uint32_t x)
425{
426 for (uint32_t i = 0; i < 32; i++) {
427 if (x & (uint32_t(1) << i)) {
428 return i;
429 }
430 }
431 return 32;
432}
433
434__forceinline uint32_t __bsr(const uint32_t x)
435{
436 for (uint32_t i = 0; i < 32; i++) {
437 if (x & (uint32_t(1) << (31 - i))) {
438 return (31 - i);
439 }
440 }
441 return 32;
442}
443
444__forceinline uint32_t __btc(const uint32_t x, const uint32_t bit)
445{
446 const uint32_t mask = uint32_t(1) << bit;
447 return x ^ mask;
448}
449
451{
452 for (uint32_t i = 0; i < 64; i++) {
453 if (x & (uint64_t(1) << i)) {
454 return i;
455 }
456 }
457 return 64;
458}
459
461{
462 for (uint32_t i = 0; i < 64; i++) {
463 if (x & (uint64_t(1) << (63 - i))) {
464 return (63 - i);
465 }
466 }
467 return 64;
468}
469
470__forceinline uint64_t __btc(const uint64_t x, const uint32_t bit)
471{
472 const uint64_t mask = uint64_t(1) << bit;
473 return x ^ mask;
474}
475
476__forceinline uint32_t bitscan(const uint32_t value)
477{
478 assert(value != 0);
479 return __bsf(value);
480}
481
483{
484 assert(value != 0);
485 return __bsf(value);
486}
487
488#endif /* Intrinsics */
489
490/* Older GCC versions do not have _mm256_cvtss_f32 yet, so define it ourselves.
491 * _mm256_castps256_ps128 generates no instructions so this is just as efficient. */
492#if defined(__KERNEL_AVX__) || defined(__KERNEL_AVX2__)
493# undef _mm256_cvtss_f32
494# define _mm256_cvtss_f32(a) (_mm_cvtss_f32(_mm256_castps256_ps128(a)))
495#endif
496
497/* quiet unused define warnings */
498#if defined(__KERNEL_SSE2__) || defined(__KERNEL_SSE3__) || defined(__KERNEL_SSSE3__) || \
499 defined(__KERNEL_SSE42__) || defined(__KERNEL_AVX__) || defined(__KERNEL_AVX2__)
500/* do nothing */
501#endif
502
unsigned long long int uint64_t
#define ccl_attr_maybe_unused
#define __forceinline
#define CCL_NAMESPACE_END
#define assert(assertion)
VecBase< float, D > step(VecOp< float, D >, VecOp< float, D >) RET
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
ATTR_WARN_UNUSED_RESULT const BMVert * v
unsigned int uint32_t
Definition BKE_text.h:81
unsigned char uint8_t
Definition BKE_text.h:79
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
#define SIMD_SET_FLUSH_TO_ZERO
__forceinline uint32_t bitscan(const uint32_t value)
Definition simd.h:476
__forceinline uint32_t __bsr(const uint32_t x)
Definition simd.h:434
__forceinline uint32_t __btc(const uint32_t x, const uint32_t bit)
Definition simd.h:444
__forceinline uint32_t __bsf(const uint32_t x)
Definition simd.h:424