Blender V5.3
simd.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2026 Blender Authors
2 *
3 * SPDX-License-Identifier: MIT */
4
8
9#pragma once
10
11#include <cassert>
12#include <cstdint>
13
14#if defined(__ARM_NEON)
15# define USE_NEON
16# include <arm_neon.h>
17#endif
18
19#if (defined(__x86_64__) || defined(_M_X64)) && defined(__SSE4_2__)
20# define USE_SSE4_2
21# include <immintrin.h>
22#endif
23
24#if defined(USE_NEON) || defined(USE_SSE4_2)
25
26namespace lexit::simd {
27
28/* Size must be power of 2. */
29template<int Size> struct u8_base {
30# if defined(USE_NEON)
31 uint8x16_t lanes[Size];
32# elif defined(USE_SSE4_2)
33 __m128i lanes[Size];
34# endif
35
36 u8_base() = default;
37
38 explicit u8_base(const uint8_t scalar)
39 {
40 for (int i = 0; i < Size; ++i) {
41# if defined(USE_NEON)
42 lanes[i] = vdupq_n_u8(scalar);
43# elif defined(USE_SSE4_2)
44 lanes[i] = _mm_set1_epi8(scalar);
45# endif
46 }
47 }
48
49 static u8_base load_unaligned(const uint8_t *src)
50 {
51 u8_base result;
52 for (int i = 0; i < Size; ++i) {
53# if defined(USE_NEON)
54 result.lanes[i] = vld1q_u8(src + i * 16);
55# elif defined(USE_SSE4_2)
56 result.lanes[i] = _mm_loadu_si128((const __m128i *)src + i);
57# endif
58 }
59 return result;
60 }
61
62 static u8_base load(const uint8_t *src)
63 {
64 assert((intptr_t(src) & (16 - 1)) == 0);
65 u8_base result;
66 for (int i = 0; i < Size; ++i) {
67# if defined(USE_NEON)
68 result.lanes[i] = vld1q_u8(src + i * 16);
69# elif defined(USE_SSE4_2)
70 result.lanes[i] = _mm_load_si128((const __m128i *)src + i);
71# endif
72 }
73 return result;
74 }
75
76 void store_unaligned(uint8_t *dst) const
77 {
78 for (int i = 0; i < Size; ++i) {
79# if defined(USE_NEON)
80 vst1q_u8(dst + i * 16, lanes[i]);
81# elif defined(USE_SSE4_2)
82 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
83# endif
84 }
85 }
86
87 void store(uint8_t *dst) const
88 {
89 for (int i = 0; i < Size; ++i) {
90# if defined(USE_NEON)
91 vst1q_u8(dst + i * 16, lanes[i]);
92# elif defined(USE_SSE4_2)
93 _mm_store_si128((__m128i *)dst + i, lanes[i]);
94# endif
95 }
96 }
97
98 u8_base<1> lane(int i) const
99 {
100 u8_base<1> result;
101 result.lanes[0] = lanes[i];
102 return result;
103 }
104
105 /* Get content of end lane */
106 uint8_t last() const
107 {
108# if defined(USE_NEON)
109 return vgetq_lane_u8(lanes[Size - 1], 15);
110# elif defined(USE_SSE4_2)
111 auto lane = lanes[Size - 1];
112 return _mm_extract_epi8(lane, 15);
113# endif
114 }
115
116 /* Get content of end lane */
117 uint8_t first() const
118 {
119# if defined(USE_NEON)
120 return vgetq_lane_u8(lanes[0], 0);
121# elif defined(USE_SSE4_2)
122 return _mm_extract_epi8(lanes[0], 0);
123# endif
124 }
125
126 /* --- Bitwise Operators --- */
127
128 friend u8_base operator^(u8_base a, u8_base b)
129 {
130 u8_base res;
131 for (int i = 0; i < Size; ++i) {
132# if defined(USE_NEON)
133 res.lanes[i] = veorq_u8(a.lanes[i], b.lanes[i]);
134# elif defined(USE_SSE4_2)
135 res.lanes[i] = _mm_xor_si128(a.lanes[i], b.lanes[i]);
136# endif
137 }
138 return res;
139 }
140
141 friend u8_base operator|(u8_base a, u8_base b)
142 {
143 u8_base res;
144 for (int i = 0; i < Size; ++i) {
145# if defined(USE_NEON)
146 res.lanes[i] = vorrq_u8(a.lanes[i], b.lanes[i]);
147# elif defined(USE_SSE4_2)
148 res.lanes[i] = _mm_or_si128(a.lanes[i], b.lanes[i]);
149# endif
150 }
151 return res;
152 }
153
154 u8_base &operator|=(u8_base b)
155 {
156 *this = *this | b;
157 return *this;
158 }
159
160 friend u8_base operator&(u8_base a, u8_base b)
161 {
162 u8_base res;
163 for (int i = 0; i < Size; ++i) {
164# if defined(USE_NEON)
165 res.lanes[i] = vandq_u8(a.lanes[i], b.lanes[i]);
166# elif defined(USE_SSE4_2)
167 res.lanes[i] = _mm_and_si128(a.lanes[i], b.lanes[i]);
168# endif
169 }
170 return res;
171 }
172
173 friend u8_base operator&(u8_base a, uint8_t b)
174 {
175# if defined(USE_NEON)
176 uint8x16_t ref = vdupq_n_u8(b);
177# elif defined(USE_SSE4_2)
178 __m128i ref = _mm_set1_epi8(b);
179# endif
180 u8_base res;
181 for (int i = 0; i < Size; ++i) {
182# if defined(USE_NEON)
183 res.lanes[i] = vandq_u8(a.lanes[i], ref);
184# elif defined(USE_SSE4_2)
185 res.lanes[i] = _mm_and_si128(a.lanes[i], ref);
186# endif
187 }
188 return res;
189 }
190
191 u8_base operator~() const
192 {
193 u8_base res;
194 for (int i = 0; i < Size; ++i) {
195# if defined(USE_NEON)
196 res.lanes[i] = vmvnq_u8(lanes[i]);
197# elif defined(USE_SSE4_2)
198 res.lanes[i] = _mm_xor_si128(lanes[i], _mm_set1_epi8(-1));
199# endif
200 }
201 return res;
202 }
203
204 /* --- Arithmetic Operators --- */
205
206 friend u8_base operator+(u8_base a, u8_base b)
207 {
208 u8_base res;
209 for (int i = 0; i < Size; ++i) {
210# if defined(USE_NEON)
211 res.lanes[i] = vaddq_u8(a.lanes[i], b.lanes[i]);
212# elif defined(USE_SSE4_2)
213 res.lanes[i] = _mm_add_epi8(a.lanes[i], b.lanes[i]);
214# endif
215 }
216 return res;
217 }
218
219 friend u8_base operator+(u8_base a, uint8_t b)
220 {
221# if defined(USE_NEON)
222 uint8x16_t ref = vdupq_n_u8(b);
223# elif defined(USE_SSE4_2)
224 __m128i ref = _mm_set1_epi8(b);
225# endif
226 u8_base res;
227 for (int i = 0; i < Size; ++i) {
228# if defined(USE_NEON)
229 res.lanes[i] = vaddq_u8(a.lanes[i], ref);
230# elif defined(USE_SSE4_2)
231 res.lanes[i] = _mm_add_epi8(a.lanes[i], ref);
232# endif
233 }
234 return res;
235 }
236
237 friend u8_base operator-(u8_base a, u8_base b)
238 {
239 u8_base res;
240 for (int i = 0; i < Size; ++i) {
241# if defined(USE_NEON)
242 res.lanes[i] = vsubq_u8(a.lanes[i], b.lanes[i]);
243# elif defined(USE_SSE4_2)
244 res.lanes[i] = _mm_sub_epi8(a.lanes[i], b.lanes[i]);
245# endif
246 }
247 return res;
248 }
249
250 /* --- Comparison Operators --- */
251
253 friend u8_base operator>(u8_base a, u8_base b)
254 {
255 u8_base res;
256 for (int i = 0; i < Size; ++i) {
257# if defined(USE_NEON)
258 res.lanes[i] = vcgtq_u8(a.lanes[i], b.lanes[i]);
259# elif defined(USE_SSE4_2)
260 res.lanes[i] = _mm_cmpgt_epi8(a.lanes[i], b.lanes[i]);
261# endif
262 }
263 return res;
264 }
265
267 friend u8_base operator<(u8_base a, u8_base b)
268 {
269 u8_base res;
270 for (int i = 0; i < Size; ++i) {
271# if defined(USE_NEON)
272 res.lanes[i] = vcltq_u8(a.lanes[i], b.lanes[i]);
273# elif defined(USE_SSE4_2)
274 res.lanes[i] = _mm_cmplt_epi8(a.lanes[i], b.lanes[i]);
275# endif
276 }
277 return res;
278 }
279
281 friend u8_base operator<(u8_base a, uint8_t b)
282 {
283# if defined(USE_NEON)
284 uint8x16_t ref = vdupq_n_u8(b);
285# elif defined(USE_SSE4_2)
286 __m128i ref = _mm_set1_epi8(b);
287# endif
288 u8_base res;
289 for (int i = 0; i < Size; ++i) {
290# if defined(USE_NEON)
291 res.lanes[i] = vcltq_u8(a.lanes[i], ref);
292# elif defined(USE_SSE4_2)
293 res.lanes[i] = _mm_cmplt_epi8(a.lanes[i], ref);
294# endif
295 }
296 return res;
297 }
298
299 friend u8_base operator==(u8_base a, uint8_t b)
300 {
301# if defined(USE_NEON)
302 uint8x16_t ref = vdupq_n_u8(b);
303# elif defined(USE_SSE4_2)
304 __m128i ref = _mm_set1_epi8(b);
305# endif
306 u8_base res;
307 for (int i = 0; i < Size; ++i) {
308# if defined(USE_NEON)
309 res.lanes[i] = vceqq_u8(a.lanes[i], ref);
310# elif defined(USE_SSE4_2)
311 res.lanes[i] = _mm_cmpeq_epi8(a.lanes[i], ref);
312# endif
313 }
314 return res;
315 }
316};
317
318using u8x16 = u8_base<1>;
319using u8x32 = u8_base<2>;
320using u8x64 = u8_base<4>;
321
322struct u8x16_table {
323 u8x16 table;
324
325 u8x16_table() = default;
326 u8x16_table(u8x16 table) : table(table) {}
327
328 static u8x16_table load_unaligned(const uint8_t *src)
329 {
330 u8x16_table table;
331 table.table = u8x16::load_unaligned(src);
332 return table;
333 }
334
335 static u8x16_table load(const uint8_t *src)
336 {
337 u8x16_table table;
338 table.table = u8x16::load(src);
339 return table;
340 }
341
342 template<int Size> u8_base<Size> operator[](u8_base<Size> index) const
343 {
344 u8_base<Size> result;
345 for (int i = 0; i < Size; ++i) {
346# if defined(USE_NEON)
347 result.lanes[i] = vqtbl1q_u8(table.lanes[0], index.lanes[i]);
348# elif defined(USE_SSE4_2)
349 /* Make sure to mimic the NEON behavior and return 0 on overflow.
350 * _mm_shuffle_epi8 will only return zero if the MSB is high. */
351 __m128i out_of_range = _mm_cmpgt_epi8(index.lanes[i], _mm_set1_epi8(15));
352 __m128i safe_indices = _mm_or_si128(index.lanes[i], out_of_range);
353 result.lanes[i] = _mm_shuffle_epi8(table.lanes[0], safe_indices);
354# endif
355 }
356 return result;
357 }
358
359 /* Only valid if all indices are less than 16. */
360 template<int Size> u8_base<Size> unsafe_shuffle(u8_base<Size> index) const
361 {
362 u8_base<Size> result;
363 for (int i = 0; i < Size; ++i) {
364# if defined(USE_NEON)
365 result.lanes[i] = vqtbl1q_u8(table.lanes[0], index.lanes[i]);
366# elif defined(USE_SSE4_2)
367 result.lanes[i] = _mm_shuffle_epi8(table.lanes[0], index.lanes[i]);
368# endif
369 }
370 return result;
371 }
372};
373
374struct u8x64_table {
375# if defined(USE_NEON)
376 uint8x16x4_t table;
377# elif defined(USE_SSE4_2)
378 u8x16_table tables[4];
379# endif
380
381 u8x64_table() = default;
382# if defined(USE_NEON)
383 u8x64_table(u8x64 table)
384 : table({table.lanes[0], table.lanes[1], table.lanes[2], table.lanes[3]})
385 {
386 }
387# elif defined(USE_SSE4_2)
388 u8x64_table(u8x64 table)
389 {
390 tables[0].table = table.lane(0);
391 tables[1].table = table.lane(1);
392 tables[2].table = table.lane(2);
393 tables[3].table = table.lane(3);
394 }
395# endif
396
397 static u8x64_table load_unaligned(const uint8_t *src)
398 {
399 u8x64_table table;
400# if defined(USE_NEON)
401 table.table = vld1q_u8_x4(src);
402# elif defined(USE_SSE4_2)
403 for (int i = 0; i < 4; ++i) {
404 table.tables[i] = u8x16_table::load_unaligned(src + i * 16);
405 }
406# endif
407 return table;
408 }
409
410 static u8x64_table load(const uint8_t *src)
411 {
412 assert((intptr_t(src) & 63) == 0);
413 u8x64_table table;
414# if defined(USE_NEON)
415 table.table = vld1q_u8_x4(src);
416# elif defined(USE_SSE4_2)
417 for (int i = 0; i < 4; ++i) {
418 table.tables[i] = u8x16_table::load(src + i * 16);
419 }
420# endif
421 return table;
422 }
423
424 template<int Size> u8_base<Size> operator[](u8_base<Size> index) const
425 {
426 u8_base<Size> result;
427# if defined(USE_NEON)
428 for (int i = 0; i < Size; ++i) {
429 result.lanes[i] = vqtbl4q_u8(table, index.lanes[i]);
430 }
431# elif defined(USE_SSE4_2)
432 result = tables[0][index];
433 result |= tables[1][index ^ u8_base<Size>(0x10)];
434 result |= tables[2][index ^ u8_base<Size>(0x20)];
435 result |= tables[3][index ^ u8_base<Size>(0x30)];
436# endif
437 return result;
438 }
439};
440
441struct u8x128_table {
442 u8x64_table tables[2];
443
444 static u8x128_table load_unaligned(const uint8_t *src)
445 {
446 u8x128_table table;
447 for (int i = 0; i < 2; ++i) {
448 table.tables[i] = u8x64_table::load_unaligned(src + i * 64);
449 }
450 return table;
451 }
452
453 static u8x128_table load(const uint8_t *src)
454 {
455 u8x128_table table;
456 for (int i = 0; i < 2; ++i) {
457 table.tables[i] = u8x64_table::load(src + i * 64);
458 }
459 return table;
460 }
461
463 template<int Size> u8_base<Size> operator[](u8_base<Size> index) const
464 {
465 /* https://lemire.me/blog/2019/07/23/arbitrary-byte-to-byte-maps-using-arm-neon/
466 * Table lookup on NEON will return 0 on overflow. Leverage this using XOR to swap which range
467 * we are looking up and combine result using OR.
468 * Note we make sure that SSE lookup have the same behavior Which is more costly
469 * (more than 3x the number of instructions) it is then preferable to avoid this path. */
470 return tables[0][index] | tables[1][index ^ u8_base<Size>(0x40)];
471 }
472};
473
474/* Select A if mask is 0, B otherwise.
475 * Mask is expected to be 0xFF or 0x00 for each component. */
476template<int Size>
477inline u8_base<Size> select(u8_base<Size> a, u8_base<Size> b, u8_base<Size> mask)
478{
479 u8_base<Size> result;
480 for (int i = 0; i < Size; ++i) {
481# if defined(USE_NEON)
482 result.lanes[i] = vbslq_u8(mask.lanes[i], b.lanes[i], a.lanes[i]);
483# elif defined(USE_SSE4_2)
484 result.lanes[i] = _mm_blendv_epi8(a.lanes[i], b.lanes[i], mask.lanes[i]);
485# endif
486 }
487 return result;
488}
489
490/* fill_value is the lanes to shift in. */
491template<int Shift, int Size>
492inline u8_base<Size> shift_lanes_right(u8_base<Size> a, uint8_t fill_value)
493{
494 u8_base<Size> result;
495 for (int i = Size - 1; i > 0; --i) {
496# if defined(USE_NEON)
497 result.lanes[i] = vextq_u8(a.lanes[i - 1], a.lanes[i], 16 - Shift);
498# elif defined(USE_SSE4_2)
499 result.lanes[i] = _mm_alignr_epi8(a.lanes[i], a.lanes[i - 1], 16 - Shift);
500# endif
501 }
502 u8_base<1> fill{fill_value};
503# if defined(USE_NEON)
504 result.lanes[0] = vextq_u8(fill.lanes[0], a.lanes[0], 16 - Shift);
505# elif defined(USE_SSE4_2)
506 result.lanes[0] = _mm_alignr_epi8(a.lanes[0], fill.lanes[0], 16 - Shift);
507# endif
508 return result;
509}
510
511template<int Size> inline u8_base<Size> is_zero(u8_base<Size> a)
512{
513# if defined(USE_SSE4_2)
514 __m128i zero = _mm_setzero_si128();
515# endif
516 u8_base<Size> result;
517 for (int i = 0; i < Size; ++i) {
518# if defined(USE_NEON)
519 result.lanes[i] = vceqzq_u8(a.lanes[i]);
520# elif defined(USE_SSE4_2)
521 result.lanes[i] = _mm_cmpeq_epi8(a.lanes[i], zero);
522# endif
523 }
524 return result;
525}
526
527/* Create a bitmask from a u8x64 mask containing 0xFF or 0x00 inside each lane. */
528inline uint64_t movemask(u8x64 mask)
529{
531
532# if defined(USE_NEON)
533 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
534 /* Merge lanes with their neighbors (e.g. [1, 2, 4, 8, ...] > [3, 12, 48, ...]).
535 * This is equivalent to merging using | and >> operations.
536 * Doing it 3 time to collapse the 8 bits. */
537 uint8x16_t sum0 = vpaddq_u8(vandq_u8(mask.lanes[0], bits), vandq_u8(mask.lanes[1], bits));
538 uint8x16_t sum1 = vpaddq_u8(vandq_u8(mask.lanes[2], bits), vandq_u8(mask.lanes[3], bits));
539 sum0 = vpaddq_u8(sum0, sum1);
540 sum0 = vpaddq_u8(sum0, sum0);
541 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
542# elif defined(USE_SSE4_2)
543 result = _mm_movemask_epi8(mask.lanes[3]);
544 result = _mm_movemask_epi8(mask.lanes[2]) | (result << 16);
545 result = _mm_movemask_epi8(mask.lanes[1]) | (result << 16);
546 result = _mm_movemask_epi8(mask.lanes[0]) | (result << 16);
547# endif
548 return result;
549}
550
551/* Create a bitmask from a u8x32 mask containing 0xFF or 0x00 inside each lane. */
552inline uint32_t movemask(u8x32 mask)
553{
555
556# if defined(USE_NEON)
557 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
558 /* Merge lanes with their neighbors (e.g. [1, 2, 4, 8, ...] > [3, 12, 48, ...]).
559 * This is equivalent to merging using | and >> operations.
560 * Doing it 3 time to collapse the 8 bits. */
561 uint8x16_t sum = vpaddq_u8(vandq_u8(mask.lanes[0], bits), vandq_u8(mask.lanes[1], bits));
562 sum = vpaddq_u8(sum, sum);
563 sum = vpaddq_u8(sum, sum);
564 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0);
565# elif defined(USE_SSE4_2)
566 result = _mm_movemask_epi8(mask.lanes[1]);
567 result = _mm_movemask_epi8(mask.lanes[0]) | (result << 16);
568# endif
569 return result;
570}
571
572/* Create a bitmask from a u8x64 mask containing 0xFF or 0x00 inside each lane. */
573inline uint16_t movemask(u8x16 mask)
574{
576
577# if defined(USE_NEON)
578 const uint8x16_t bits = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128};
579 uint8x16_t sum = vandq_u8(mask.lanes[0], bits);
580 /* Merge lanes with their neighbors (e.g. [1, 2, 4, 8, ...] > [3, 12, 48, ...]).
581 * This is equivalent to merging using | and >> operations.
582 * Doing it 3 time to collapse the 8 bits. */
583 sum = vpaddq_u8(sum, sum);
584 sum = vpaddq_u8(sum, sum);
585 sum = vpaddq_u8(sum, sum);
586 result = vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0);
587# elif defined(USE_SSE4_2)
588 result = _mm_movemask_epi8(mask.lanes[0]);
589# endif
590 return result;
591}
592
593/* Size must be power of 2. */
594template<int Size> struct u16_base {
595# if defined(USE_NEON)
596 uint16x8_t lanes[Size];
597# elif defined(USE_SSE4_2)
598 __m128i lanes[Size];
599# endif
600
601 u16_base() = default;
602
603 explicit u16_base(const u8_base<Size / 2> v)
604 {
605# if defined(USE_SSE4_2)
606 __m128i zero = _mm_setzero_si128();
607# endif
608 for (int i = 0; i < Size / 2; ++i) {
609# if defined(USE_NEON)
610 lanes[i * 2 + 0] = vmovl_u8(vget_low_u8(v.lanes[i]));
611 lanes[i * 2 + 1] = vmovl_u8(vget_high_u8(v.lanes[i]));
612# elif defined(USE_SSE4_2)
613 lanes[i * 2 + 0] = _mm_unpacklo_epi8(v.lanes[i], zero);
614 lanes[i * 2 + 1] = _mm_unpackhi_epi8(v.lanes[i], zero);
615# endif
616 }
617 }
618
619 void store(uint16_t *dst) const
620 {
621 for (int i = 0; i < Size; ++i) {
622# if defined(USE_NEON)
623 vst1q_u16(dst + i * 8, lanes[i]);
624# elif defined(USE_SSE4_2)
625 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
626# endif
627 }
628 }
629};
630
631using u16x16 = u16_base<2>;
632using u16x64 = u16_base<8>;
633
634/* Size must be power of 2. */
635template<int Size> struct u32_base {
636# if defined(USE_NEON)
637 uint32x4_t lanes[Size];
638# elif defined(USE_SSE4_2)
639 __m128i lanes[Size];
640# endif
641
642 u32_base() = default;
643
644 explicit u32_base(const u8_base<Size / 4> v)
645 {
646 for (int i = 0; i < Size / 4; ++i) {
647# if defined(USE_NEON)
648 uint16x8_t tmp_lo = vmovl_u8(vget_low_u8(v.lanes[i]));
649 uint16x8_t tmp_hi = vmovl_u8(vget_high_u8(v.lanes[i]));
650 lanes[i * 4 + 0] = vmovl_u16(vget_low_u16(tmp_lo));
651 lanes[i * 4 + 1] = vmovl_u16(vget_high_u16(tmp_lo));
652 lanes[i * 4 + 2] = vmovl_u16(vget_low_u16(tmp_hi));
653 lanes[i * 4 + 3] = vmovl_u16(vget_high_u16(tmp_hi));
654# elif defined(USE_SSE4_2)
655 lanes[i * 4 + 0] = _mm_cvtepu8_epi32(v.lanes[i]);
656 lanes[i * 4 + 1] = _mm_cvtepu8_epi32(_mm_srli_si128(v.lanes[i], 4));
657 lanes[i * 4 + 2] = _mm_cvtepu8_epi32(_mm_srli_si128(v.lanes[i], 8));
658 lanes[i * 4 + 3] = _mm_cvtepu8_epi32(_mm_srli_si128(v.lanes[i], 12));
659# endif
660 }
661 }
662
663 /* NOTE: This does a signed saturation on SSE. */
664 explicit operator u8_base<Size / 4>() const
665 {
666 u8_base<Size / 4> res;
667 for (int i = 0; i < Size / 4; ++i) {
668# if defined(USE_NEON)
669 uint16x4_t n0 = vmovn_u32(lanes[i * 4 + 0]);
670 uint16x4_t n1 = vmovn_u32(lanes[i * 4 + 1]);
671 uint16x4_t n2 = vmovn_u32(lanes[i * 4 + 2]);
672 uint16x4_t n3 = vmovn_u32(lanes[i * 4 + 3]);
673 uint16x8_t q01 = vcombine_u16(n0, n1);
674 uint16x8_t q23 = vcombine_u16(n2, n3);
675 res.lanes[i] = vcombine_u8(vmovn_u16(q01), vmovn_u16(q23));
676# elif defined(USE_SSE4_2)
677 __m128i pack_16_lo = _mm_packs_epi32(lanes[i * 4 + 0], lanes[i * 4 + 1]);
678 __m128i pack_16_hi = _mm_packs_epi32(lanes[i * 4 + 2], lanes[i * 4 + 3]);
679 res.lanes[i] = _mm_packs_epi16(pack_16_lo, pack_16_hi);
680# endif
681 }
682 return res;
683 }
684
685 static u32_base load_unaligned(const uint32_t *src)
686 {
687 u32_base result;
688 for (int i = 0; i < Size; ++i) {
689# if defined(USE_NEON)
690 result.lanes[i] = vld1q_u32(src + i * 4);
691# elif defined(USE_SSE4_2)
692 result.lanes[i] = _mm_loadu_si128((const __m128i *)src + i);
693# endif
694 }
695 return result;
696 }
697
698 static u32_base load(const uint32_t *src)
699 {
700 assert((intptr_t(src) & (16 - 1)) == 0);
701 u32_base result;
702 for (int i = 0; i < Size; ++i) {
703# if defined(USE_NEON)
704 result.lanes[i] = vld1q_u32(src + i * 4);
705# elif defined(USE_SSE4_2)
706 result.lanes[i] = _mm_load_si128((const __m128i *)src + i);
707# endif
708 }
709 return result;
710 }
711
712 void store_unaligned(uint32_t *dst) const
713 {
714 for (int i = 0; i < Size; ++i) {
715# if defined(USE_NEON)
716 vst1q_u32(dst + i * 4, lanes[i]);
717# elif defined(USE_SSE4_2)
718 _mm_storeu_si128((__m128i *)dst + i, lanes[i]);
719# endif
720 }
721 }
722
723 void store(uint32_t *dst) const
724 {
725 assert((intptr_t(dst) & (16 - 1)) == 0);
726 for (int i = 0; i < Size; ++i) {
727# if defined(USE_NEON)
728 vst1q_u32(dst + i * 4, lanes[i]);
729# elif defined(USE_SSE4_2)
730 _mm_store_si128((__m128i *)dst + i, lanes[i]);
731# endif
732 }
733 }
734
735 /* --- Arithmetic Operators --- */
736
737 /* Note: Signed comparison on SSE. */
738 friend u32_base operator<(u32_base a, u32_base b)
739 {
740 u32_base res;
741 for (int i = 0; i < Size; ++i) {
742# if defined(USE_NEON)
743 res.lanes[i] = vcltq_u32(a.lanes[i], b.lanes[i]);
744# elif defined(USE_SSE4_2)
745 res.lanes[i] = _mm_cmplt_epi32(a.lanes[i], b.lanes[i]);
746# endif
747 }
748 return res;
749 }
750
751 /* Note: Signed comparison on SSE. */
752 friend u32_base operator<(u32_base a, uint32_t b)
753 {
754# if defined(USE_NEON)
755 uint32x4_t tmp = vdupq_n_u32(b);
756# elif defined(USE_SSE4_2)
757 __m128i tmp = _mm_set1_epi32(b);
758# endif
759
760 u32_base res;
761 for (int i = 0; i < Size; ++i) {
762# if defined(USE_NEON)
763 res.lanes[i] = vcltq_u32(a.lanes[i], tmp);
764# elif defined(USE_SSE4_2)
765 res.lanes[i] = _mm_cmplt_epi32(a.lanes[i], tmp);
766# endif
767 }
768 return res;
769 }
770
771 friend u32_base operator>(u32_base a, uint32_t b)
772 {
773# if defined(USE_NEON)
774 uint32x4_t tmp = vdupq_n_u32(b);
775# elif defined(USE_SSE4_2)
776 __m128i tmp = _mm_set1_epi32(b);
777# endif
778
779 u32_base res;
780 for (int i = 0; i < Size; ++i) {
781# if defined(USE_NEON)
782 res.lanes[i] = vcgtq_u32(a.lanes[i], tmp);
783# elif defined(USE_SSE4_2)
784 res.lanes[i] = _mm_cmpgt_epi32(a.lanes[i], tmp);
785# endif
786 }
787 return res;
788 }
789
790 friend u32_base operator+(u32_base a, uint32_t b)
791 {
792# if defined(USE_NEON)
793 uint32x4_t tmp = vdupq_n_u32(b);
794# elif defined(USE_SSE4_2)
795 __m128i tmp = _mm_set1_epi32(b);
796# endif
797
798 u32_base res;
799 for (int i = 0; i < Size; ++i) {
800# if defined(USE_NEON)
801 res.lanes[i] = vaddq_u32(a.lanes[i], tmp);
802# elif defined(USE_SSE4_2)
803 res.lanes[i] = _mm_add_epi32(a.lanes[i], tmp);
804# endif
805 }
806 return res;
807 }
808
809 friend u32_base operator-(u32_base a, u32_base b)
810 {
811 u32_base res;
812 for (int i = 0; i < Size; ++i) {
813# if defined(USE_NEON)
814 res.lanes[i] = vsubq_u32(a.lanes[i], b.lanes[i]);
815# elif defined(USE_SSE4_2)
816 res.lanes[i] = _mm_sub_epi32(a.lanes[i], b.lanes[i]);
817# endif
818 }
819 return res;
820 }
821};
822
823using u32x16 = u32_base<4>;
824using u32x32 = u32_base<8>;
825using u32x64 = u32_base<16>;
826
827} // namespace lexit::simd
828
829#endif
unsigned long long int uint64_t
SIMD_FORCE_INLINE btVector3 & operator[](int i)
Get a mutable reference to a row of the matrix as a vector.
#define assert(assertion)
#define index(i)
#define select(A, B, C)
bool is_zero(float3x3 a)
__forceinline LightTreeMeasure operator+(const LightTreeMeasure &a, const LightTreeMeasure &b)
ccl_device_inline dual< T1 > operator-(const dual< T1 > a, T2 b)
Definition math_dual.h:113
ccl_device_inline bool operator<(const ccl_private dual1 &a, const float b)
Definition math_dual.h:45
ccl_device_inline bool operator==(const float2 a, const float2 b)
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
ccl_device_inline int3 operator>(const float3 a, const float3 b)
ccl_device_inline float4 operator^(const float4 a, const float4 b)
ccl_device_inline int2 operator&(const int2 a, const int2 b)
Definition math_int2.h:48
ccl_device_inline int4 & operator|=(int4 &a, const int4 b)
Definition math_int4.h:178
ccl_device_inline int4 operator|(const int4 a, const int4 b)
Definition math_int4.h:121
void load(const VolumeGridData &grid)
GPUState operator~(const GPUState &a)
BLI_NOINLINE void fill(T *__restrict data, const T &value, const IndexMask &mask)
ATTR_WARN_UNUSED_RESULT const BMVert * v
unsigned short uint16_t
Definition BKE_text.h:80
unsigned int uint32_t
Definition BKE_text.h:81
unsigned char uint8_t
Definition BKE_text.h:79
_W64 int intptr_t
Definition BKE_text.h:119