14#if defined(__ARM_NEON)
19#if (defined(__x86_64__) || defined(_M_X64)) && defined(__SSE4_2__)
21# include <immintrin.h>
24#if defined(USE_NEON) || defined(USE_SSE4_2)
26namespace lexit::simd {
29template<
int Size>
struct u8_base {
31 uint8x16_t lanes[Size];
32# elif defined(USE_SSE4_2)
38 explicit u8_base(
const uint8_t scalar)
40 for (
int i = 0; i < Size; ++i) {
42 lanes[i] = vdupq_n_u8(scalar);
43# elif defined(USE_SSE4_2)
44 lanes[i] = _mm_set1_epi8(scalar);
49 static u8_base load_unaligned(
const uint8_t *src)
52 for (
int i = 0; i < Size; ++i) {
54 result.lanes[i] = vld1q_u8(src + i * 16);
55# elif defined(USE_SSE4_2)
56 result.lanes[i] = _mm_loadu_si128((
const __m128i *)src + i);
62 static u8_base
load(
const uint8_t *src)
66 for (
int i = 0; i < Size; ++i) {
68 result.lanes[i] = vld1q_u8(src + i * 16);
69# elif defined(USE_SSE4_2)
70 result.lanes[i] = _mm_load_si128((
const __m128i *)src + i);
76 void store_unaligned(uint8_t *dst)
const
78 for (
int i = 0; i < Size; ++i) {
80 vst1q_u8(dst + i * 16, lanes[i]);
81# elif defined(USE_SSE4_2)
82 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
87 void store(uint8_t *dst)
const
89 for (
int i = 0; i < Size; ++i) {
91 vst1q_u8(dst + i * 16, lanes[i]);
92# elif defined(USE_SSE4_2)
93 _mm_store_si128((__m128i *)dst + i, lanes[i]);
98 u8_base<1> lane(
int i)
const
101 result.lanes[0] = lanes[i];
108# if defined(USE_NEON)
109 return vgetq_lane_u8(lanes[Size - 1], 15);
110# elif defined(USE_SSE4_2)
111 auto lane = lanes[Size - 1];
112 return _mm_extract_epi8(lane, 15);
119# if defined(USE_NEON)
120 return vgetq_lane_u8(lanes[0], 0);
121# elif defined(USE_SSE4_2)
122 return _mm_extract_epi8(lanes[0], 0);
128 friend u8_base
operator^(u8_base a, u8_base
b)
131 for (
int i = 0; i < Size; ++i) {
132# if defined(USE_NEON)
133 res.lanes[i] = veorq_u8(a.lanes[i],
b.lanes[i]);
134# elif defined(USE_SSE4_2)
135 res.lanes[i] = _mm_xor_si128(a.lanes[i],
b.lanes[i]);
141 friend u8_base
operator|(u8_base a, u8_base
b)
144 for (
int i = 0; i < Size; ++i) {
145# if defined(USE_NEON)
146 res.lanes[i] = vorrq_u8(a.lanes[i],
b.lanes[i]);
147# elif defined(USE_SSE4_2)
148 res.lanes[i] = _mm_or_si128(a.lanes[i],
b.lanes[i]);
160 friend u8_base
operator&(u8_base a, u8_base
b)
163 for (
int i = 0; i < Size; ++i) {
164# if defined(USE_NEON)
165 res.lanes[i] = vandq_u8(a.lanes[i],
b.lanes[i]);
166# elif defined(USE_SSE4_2)
167 res.lanes[i] = _mm_and_si128(a.lanes[i],
b.lanes[i]);
173 friend u8_base
operator&(u8_base a, uint8_t
b)
175# if defined(USE_NEON)
176 uint8x16_t ref = vdupq_n_u8(
b);
177# elif defined(USE_SSE4_2)
178 __m128i ref = _mm_set1_epi8(
b);
181 for (
int i = 0; i < Size; ++i) {
182# if defined(USE_NEON)
183 res.lanes[i] = vandq_u8(a.lanes[i], ref);
184# elif defined(USE_SSE4_2)
185 res.lanes[i] = _mm_and_si128(a.lanes[i], ref);
194 for (
int i = 0; i < Size; ++i) {
195# if defined(USE_NEON)
196 res.lanes[i] = vmvnq_u8(lanes[i]);
197# elif defined(USE_SSE4_2)
198 res.lanes[i] = _mm_xor_si128(lanes[i], _mm_set1_epi8(-1));
206 friend u8_base
operator+(u8_base a, u8_base
b)
209 for (
int i = 0; i < Size; ++i) {
210# if defined(USE_NEON)
211 res.lanes[i] = vaddq_u8(a.lanes[i],
b.lanes[i]);
212# elif defined(USE_SSE4_2)
213 res.lanes[i] = _mm_add_epi8(a.lanes[i],
b.lanes[i]);
219 friend u8_base
operator+(u8_base a, uint8_t
b)
221# if defined(USE_NEON)
222 uint8x16_t ref = vdupq_n_u8(
b);
223# elif defined(USE_SSE4_2)
224 __m128i ref = _mm_set1_epi8(
b);
227 for (
int i = 0; i < Size; ++i) {
228# if defined(USE_NEON)
229 res.lanes[i] = vaddq_u8(a.lanes[i], ref);
230# elif defined(USE_SSE4_2)
231 res.lanes[i] = _mm_add_epi8(a.lanes[i], ref);
237 friend u8_base
operator-(u8_base a, u8_base
b)
240 for (
int i = 0; i < Size; ++i) {
241# if defined(USE_NEON)
242 res.lanes[i] = vsubq_u8(a.lanes[i],
b.lanes[i]);
243# elif defined(USE_SSE4_2)
244 res.lanes[i] = _mm_sub_epi8(a.lanes[i],
b.lanes[i]);
253 friend u8_base
operator>(u8_base a, u8_base
b)
256 for (
int i = 0; i < Size; ++i) {
257# if defined(USE_NEON)
258 res.lanes[i] = vcgtq_u8(a.lanes[i],
b.lanes[i]);
259# elif defined(USE_SSE4_2)
260 res.lanes[i] = _mm_cmpgt_epi8(a.lanes[i],
b.lanes[i]);
267 friend u8_base
operator<(u8_base a, u8_base
b)
270 for (
int i = 0; i < Size; ++i) {
271# if defined(USE_NEON)
272 res.lanes[i] = vcltq_u8(a.lanes[i],
b.lanes[i]);
273# elif defined(USE_SSE4_2)
274 res.lanes[i] = _mm_cmplt_epi8(a.lanes[i],
b.lanes[i]);
281 friend u8_base
operator<(u8_base a, uint8_t
b)
283# if defined(USE_NEON)
284 uint8x16_t ref = vdupq_n_u8(
b);
285# elif defined(USE_SSE4_2)
286 __m128i ref = _mm_set1_epi8(
b);
289 for (
int i = 0; i < Size; ++i) {
290# if defined(USE_NEON)
291 res.lanes[i] = vcltq_u8(a.lanes[i], ref);
292# elif defined(USE_SSE4_2)
293 res.lanes[i] = _mm_cmplt_epi8(a.lanes[i], ref);
301# if defined(USE_NEON)
302 uint8x16_t ref = vdupq_n_u8(
b);
303# elif defined(USE_SSE4_2)
304 __m128i ref = _mm_set1_epi8(
b);
307 for (
int i = 0; i < Size; ++i) {
308# if defined(USE_NEON)
309 res.lanes[i] = vceqq_u8(a.lanes[i], ref);
310# elif defined(USE_SSE4_2)
311 res.lanes[i] = _mm_cmpeq_epi8(a.lanes[i], ref);
318using u8x16 = u8_base<1>;
319using u8x32 = u8_base<2>;
320using u8x64 = u8_base<4>;
325 u8x16_table() =
default;
326 u8x16_table(u8x16 table) : table(table) {}
328 static u8x16_table load_unaligned(
const uint8_t *src)
331 table.table = u8x16::load_unaligned(src);
335 static u8x16_table
load(
const uint8_t *src)
338 table.table = u8x16::load(src);
345 for (
int i = 0; i < Size; ++i) {
346# if defined(USE_NEON)
347 result.lanes[i] = vqtbl1q_u8(table.lanes[0],
index.lanes[i]);
348# elif defined(USE_SSE4_2)
351 __m128i out_of_range = _mm_cmpgt_epi8(
index.lanes[i], _mm_set1_epi8(15));
352 __m128i safe_indices = _mm_or_si128(
index.lanes[i], out_of_range);
353 result.lanes[i] = _mm_shuffle_epi8(table.lanes[0], safe_indices);
360 template<
int Size> u8_base<Size> unsafe_shuffle(u8_base<Size>
index)
const
363 for (
int i = 0; i < Size; ++i) {
364# if defined(USE_NEON)
365 result.lanes[i] = vqtbl1q_u8(table.lanes[0],
index.lanes[i]);
366# elif defined(USE_SSE4_2)
367 result.lanes[i] = _mm_shuffle_epi8(table.lanes[0],
index.lanes[i]);
375# if defined(USE_NEON)
377# elif defined(USE_SSE4_2)
378 u8x16_table tables[4];
381 u8x64_table() =
default;
382# if defined(USE_NEON)
383 u8x64_table(u8x64 table)
384 : table({table.lanes[0], table.lanes[1], table.lanes[2], table.lanes[3]})
387# elif defined(USE_SSE4_2)
388 u8x64_table(u8x64 table)
390 tables[0].table = table.lane(0);
391 tables[1].table = table.lane(1);
392 tables[2].table = table.lane(2);
393 tables[3].table = table.lane(3);
397 static u8x64_table load_unaligned(
const uint8_t *src)
400# if defined(USE_NEON)
401 table.table = vld1q_u8_x4(src);
402# elif defined(USE_SSE4_2)
403 for (
int i = 0; i < 4; ++i) {
404 table.tables[i] = u8x16_table::load_unaligned(src + i * 16);
410 static u8x64_table
load(
const uint8_t *src)
414# if defined(USE_NEON)
415 table.table = vld1q_u8_x4(src);
416# elif defined(USE_SSE4_2)
417 for (
int i = 0; i < 4; ++i) {
418 table.tables[i] = u8x16_table::load(src + i * 16);
427# if defined(USE_NEON)
428 for (
int i = 0; i < Size; ++i) {
429 result.lanes[i] = vqtbl4q_u8(table,
index.lanes[i]);
431# elif defined(USE_SSE4_2)
442 u8x64_table tables[2];
444 static u8x128_table load_unaligned(
const uint8_t *src)
447 for (
int i = 0; i < 2; ++i) {
448 table.tables[i] = u8x64_table::load_unaligned(src + i * 64);
453 static u8x128_table
load(
const uint8_t *src)
456 for (
int i = 0; i < 2; ++i) {
457 table.tables[i] = u8x64_table::load(src + i * 64);
470 return tables[0][
index] | tables[1][
index ^ u8_base<Size>(0x40)];
477inline u8_base<Size>
select(u8_base<Size> a, u8_base<Size>
b, u8_base<Size>
mask)
480 for (
int i = 0; i < Size; ++i) {
481# if defined(USE_NEON)
482 result.lanes[i] = vbslq_u8(
mask.lanes[i],
b.lanes[i], a.lanes[i]);
483# elif defined(USE_SSE4_2)
484 result.lanes[i] = _mm_blendv_epi8(a.lanes[i],
b.lanes[i],
mask.lanes[i]);
491template<
int Shift,
int Size>
492inline u8_base<Size> shift_lanes_right(u8_base<Size> a, uint8_t fill_value)
495 for (
int i = Size - 1; i > 0; --i) {
496# if defined(USE_NEON)
497 result.lanes[i] = vextq_u8(a.lanes[i - 1], a.lanes[i], 16 - Shift);
498# elif defined(USE_SSE4_2)
499 result.lanes[i] = _mm_alignr_epi8(a.lanes[i], a.lanes[i - 1], 16 - Shift);
502 u8_base<1>
fill{fill_value};
503# if defined(USE_NEON)
504 result.lanes[0] = vextq_u8(
fill.lanes[0], a.lanes[0], 16 - Shift);
505# elif defined(USE_SSE4_2)
506 result.lanes[0] = _mm_alignr_epi8(a.lanes[0],
fill.lanes[0], 16 - Shift);
511template<
int Size>
inline u8_base<Size>
is_zero(u8_base<Size> a)
513# if defined(USE_SSE4_2)
514 __m128i zero = _mm_setzero_si128();
517 for (
int i = 0; i < Size; ++i) {
518# if defined(USE_NEON)
519 result.lanes[i] = vceqzq_u8(a.lanes[i]);
520# elif defined(USE_SSE4_2)
521 result.lanes[i] = _mm_cmpeq_epi8(a.lanes[i], zero);
532# if defined(USE_NEON)
533 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
537 uint8x16_t sum0 = vpaddq_u8(vandq_u8(
mask.lanes[0], bits), vandq_u8(
mask.lanes[1], bits));
538 uint8x16_t sum1 = vpaddq_u8(vandq_u8(
mask.lanes[2], bits), vandq_u8(
mask.lanes[3], bits));
539 sum0 = vpaddq_u8(sum0, sum1);
540 sum0 = vpaddq_u8(sum0, sum0);
541 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
542# elif defined(USE_SSE4_2)
556# if defined(USE_NEON)
557 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
561 uint8x16_t sum = vpaddq_u8(vandq_u8(
mask.lanes[0], bits), vandq_u8(
mask.lanes[1], bits));
562 sum = vpaddq_u8(sum, sum);
563 sum = vpaddq_u8(sum, sum);
564 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0);
565# elif defined(USE_SSE4_2)
577# if defined(USE_NEON)
578 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
579 uint8x16_t sum = vandq_u8(
mask.lanes[0], bits);
583 sum = vpaddq_u8(sum, sum);
584 sum = vpaddq_u8(sum, sum);
585 sum = vpaddq_u8(sum, sum);
586 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0);
587# elif defined(USE_SSE4_2)
594template<
int Size>
struct u16_base {
595# if defined(USE_NEON)
596 uint16x8_t lanes[Size];
597# elif defined(USE_SSE4_2)
601 u16_base() =
default;
603 explicit u16_base(
const u8_base<Size / 2> v)
605# if defined(USE_SSE4_2)
606 __m128i zero = _mm_setzero_si128();
608 for (
int i = 0; i < Size / 2; ++i) {
609# if defined(USE_NEON)
610 lanes[i * 2 + 0] = vmovl_u8(vget_low_u8(
v.lanes[i]));
611 lanes[i * 2 + 1] = vmovl_u8(vget_high_u8(
v.lanes[i]));
612# elif defined(USE_SSE4_2)
613 lanes[i * 2 + 0] = _mm_unpacklo_epi8(
v.lanes[i], zero);
614 lanes[i * 2 + 1] = _mm_unpackhi_epi8(
v.lanes[i], zero);
619 void store(uint16_t *dst)
const
621 for (
int i = 0; i < Size; ++i) {
622# if defined(USE_NEON)
623 vst1q_u16(dst + i * 8, lanes[i]);
624# elif defined(USE_SSE4_2)
625 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
631using u16x16 = u16_base<2>;
632using u16x64 = u16_base<8>;
635template<
int Size>
struct u32_base {
636# if defined(USE_NEON)
637 uint32x4_t lanes[Size];
638# elif defined(USE_SSE4_2)
642 u32_base() =
default;
644 explicit u32_base(
const u8_base<Size / 4> v)
646 for (
int i = 0; i < Size / 4; ++i) {
647# if defined(USE_NEON)
648 uint16x8_t tmp_lo = vmovl_u8(vget_low_u8(
v.lanes[i]));
649 uint16x8_t tmp_hi = vmovl_u8(vget_high_u8(
v.lanes[i]));
650 lanes[i * 4 + 0] = vmovl_u16(vget_low_u16(tmp_lo));
651 lanes[i * 4 + 1] = vmovl_u16(vget_high_u16(tmp_lo));
652 lanes[i * 4 + 2] = vmovl_u16(vget_low_u16(tmp_hi));
653 lanes[i * 4 + 3] = vmovl_u16(vget_high_u16(tmp_hi));
654# elif defined(USE_SSE4_2)
655 lanes[i * 4 + 0] = _mm_cvtepu8_epi32(
v.lanes[i]);
656 lanes[i * 4 + 1] = _mm_cvtepu8_epi32(_mm_srli_si128(
v.lanes[i], 4));
657 lanes[i * 4 + 2] = _mm_cvtepu8_epi32(_mm_srli_si128(
v.lanes[i], 8));
658 lanes[i * 4 + 3] = _mm_cvtepu8_epi32(_mm_srli_si128(
v.lanes[i], 12));
664 explicit operator u8_base<Size / 4>()
const
666 u8_base<Size / 4> res;
667 for (
int i = 0; i < Size / 4; ++i) {
668# if defined(USE_NEON)
669 uint16x4_t n0 = vmovn_u32(lanes[i * 4 + 0]);
670 uint16x4_t n1 = vmovn_u32(lanes[i * 4 + 1]);
671 uint16x4_t n2 = vmovn_u32(lanes[i * 4 + 2]);
672 uint16x4_t n3 = vmovn_u32(lanes[i * 4 + 3]);
673 uint16x8_t q01 = vcombine_u16(n0, n1);
674 uint16x8_t q23 = vcombine_u16(n2, n3);
675 res.lanes[i] = vcombine_u8(vmovn_u16(q01), vmovn_u16(q23));
676# elif defined(USE_SSE4_2)
677 __m128i pack_16_lo = _mm_packs_epi32(lanes[i * 4 + 0], lanes[i * 4 + 1]);
678 __m128i pack_16_hi = _mm_packs_epi32(lanes[i * 4 + 2], lanes[i * 4 + 3]);
679 res.lanes[i] = _mm_packs_epi16(pack_16_lo, pack_16_hi);
685 static u32_base load_unaligned(
const uint32_t *src)
688 for (
int i = 0; i < Size; ++i) {
689# if defined(USE_NEON)
690 result.lanes[i] = vld1q_u32(src + i * 4);
691# elif defined(USE_SSE4_2)
692 result.lanes[i] = _mm_loadu_si128((
const __m128i *)src + i);
698 static u32_base
load(
const uint32_t *src)
702 for (
int i = 0; i < Size; ++i) {
703# if defined(USE_NEON)
704 result.lanes[i] = vld1q_u32(src + i * 4);
705# elif defined(USE_SSE4_2)
706 result.lanes[i] = _mm_load_si128((
const __m128i *)src + i);
712 void store_unaligned(uint32_t *dst)
const
714 for (
int i = 0; i < Size; ++i) {
715# if defined(USE_NEON)
716 vst1q_u32(dst + i * 4, lanes[i]);
717# elif defined(USE_SSE4_2)
718 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
723 void store(uint32_t *dst)
const
726 for (
int i = 0; i < Size; ++i) {
727# if defined(USE_NEON)
728 vst1q_u32(dst + i * 4, lanes[i]);
729# elif defined(USE_SSE4_2)
730 _mm_store_si128((__m128i *)dst + i, lanes[i]);
738 friend u32_base
operator<(u32_base a, u32_base
b)
741 for (
int i = 0; i < Size; ++i) {
742# if defined(USE_NEON)
743 res.lanes[i] = vcltq_u32(a.lanes[i],
b.lanes[i]);
744# elif defined(USE_SSE4_2)
745 res.lanes[i] = _mm_cmplt_epi32(a.lanes[i],
b.lanes[i]);
752 friend u32_base
operator<(u32_base a, uint32_t
b)
754# if defined(USE_NEON)
755 uint32x4_t tmp = vdupq_n_u32(
b);
756# elif defined(USE_SSE4_2)
757 __m128i tmp = _mm_set1_epi32(
b);
761 for (
int i = 0; i < Size; ++i) {
762# if defined(USE_NEON)
763 res.lanes[i] = vcltq_u32(a.lanes[i], tmp);
764# elif defined(USE_SSE4_2)
765 res.lanes[i] = _mm_cmplt_epi32(a.lanes[i], tmp);
771 friend u32_base
operator>(u32_base a, uint32_t
b)
773# if defined(USE_NEON)
774 uint32x4_t tmp = vdupq_n_u32(
b);
775# elif defined(USE_SSE4_2)
776 __m128i tmp = _mm_set1_epi32(
b);
780 for (
int i = 0; i < Size; ++i) {
781# if defined(USE_NEON)
782 res.lanes[i] = vcgtq_u32(a.lanes[i], tmp);
783# elif defined(USE_SSE4_2)
784 res.lanes[i] = _mm_cmpgt_epi32(a.lanes[i], tmp);
790 friend u32_base
operator+(u32_base a, uint32_t
b)
792# if defined(USE_NEON)
793 uint32x4_t tmp = vdupq_n_u32(
b);
794# elif defined(USE_SSE4_2)
795 __m128i tmp = _mm_set1_epi32(
b);
799 for (
int i = 0; i < Size; ++i) {
800# if defined(USE_NEON)
801 res.lanes[i] = vaddq_u32(a.lanes[i], tmp);
802# elif defined(USE_SSE4_2)
803 res.lanes[i] = _mm_add_epi32(a.lanes[i], tmp);
809 friend u32_base
operator-(u32_base a, u32_base
b)
812 for (
int i = 0; i < Size; ++i) {
813# if defined(USE_NEON)
814 res.lanes[i] = vsubq_u32(a.lanes[i],
b.lanes[i]);
815# elif defined(USE_SSE4_2)
816 res.lanes[i] = _mm_sub_epi32(a.lanes[i],
b.lanes[i]);
823using u32x16 = u32_base<4>;
824using u32x32 = u32_base<8>;
825using u32x64 = u32_base<16>;
unsigned long long int uint64_t
SIMD_FORCE_INLINE btVector3 & operator[](int i)
Get a mutable reference to a row of the matrix as a vector.
#define assert(assertion)
__forceinline LightTreeMeasure operator+(const LightTreeMeasure &a, const LightTreeMeasure &b)
ccl_device_inline dual< T1 > operator-(const dual< T1 > a, T2 b)
ccl_device_inline bool operator<(const ccl_private dual1 &a, const float b)
ccl_device_inline bool operator==(const float2 a, const float2 b)
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
ccl_device_inline int3 operator>(const float3 a, const float3 b)
ccl_device_inline float4 operator^(const float4 a, const float4 b)
ccl_device_inline int2 operator&(const int2 a, const int2 b)
ccl_device_inline int4 & operator|=(int4 &a, const int4 b)
ccl_device_inline int4 operator|(const int4 a, const int4 b)
void load(const VolumeGridData &grid)
GPUState operator~(const GPUState &a)
BLI_NOINLINE void fill(T *__restrict data, const T &value, const IndexMask &mask)
ATTR_WARN_UNUSED_RESULT const BMVert * v