13#if defined(__ARM_NEON)
15# define USE_HARDWARE_FP16_NEON
18#if (defined(__x86_64__) || defined(_M_X64))
21# if (defined(__AVX__) && defined(__F16C__)) || defined(__AVX2__)
22# define USE_HARDWARE_FP16_F16C
27# include <immintrin.h>
34#if defined(USE_HARDWARE_FP16_NEON)
35 float16x4_t h4 = vcvt_f16_f32(vdupq_n_f32(
v));
36 float16_t h = vget_lane_f16(h4, 0);
37 return *
reinterpret_cast<uint16_t *
>(&h);
48 FP32 f32infty = {255 << 23};
49 FP32 f16max = {(127 + 16) << 23};
50 FP32 denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23};
63 if (f.u >= f16max.u) {
65 o = (f.u > f32infty.u) ? 0x7e00 : 0x7c00;
69 if (f.u < (113 << 23)) {
74 f.f += denorm_magic.f;
77 o = f.u - denorm_magic.u;
83 f.u += (
uint32_t(15 - 127) << 23) + 0xfff;
100 if ((h & 0x7c00) == 0x7c00) {
101 if ((h & 0x03ff) == 0) {
115#if defined(USE_HARDWARE_FP16_NEON)
116 uint16x4_t v4 = vdup_n_u16(
v);
117 float16x4_t h4 = vreinterpret_f16_u16(v4);
118 float32x4_t f4 = vcvt_f32_f16(h4);
119 return vgetq_lane_f32(f4, 0);
128 constexpr FP32 magic = {113 << 23};
129 constexpr uint32_t shifted_exp = 0x7c00 << 13;
132 o.u = (
v & 0x7fff) << 13;
134 o.u += (127 - 15) << 23;
137 if (
exp == shifted_exp) {
138 o.u += (128 - 16) << 23;
145 o.u |= (
v & 0x8000) << 16;
156static inline __m128i F32_to_F16_4x(
const __m128 &f)
158 const __m128 mask_sign = _mm_set1_ps(-0.0f);
160 const __m128i c_f16max = _mm_set1_epi32((127 + 16) << 23);
161 const __m128i c_nanbit = _mm_set1_epi32(0x200);
162 const __m128i c_nanlobits = _mm_set1_epi32(0x1ff);
163 const __m128i c_infty_as_fp16 = _mm_set1_epi32(0x7c00);
165 const __m128i c_min_normal = _mm_set1_epi32((127 - 14) << 23);
166 const __m128i c_subnorm_magic = _mm_set1_epi32(((127 - 15) + (23 - 10) + 1) << 23);
168 const __m128i c_normal_bias = _mm_set1_epi32(0xfff - ((127 - 15) << 23));
170 __m128 justsign = _mm_and_ps(f, mask_sign);
171 __m128 absf = _mm_andnot_ps(mask_sign, f);
173 __m128i absf_int = _mm_castps_si128(absf);
174 __m128 b_isnan = _mm_cmpunord_ps(absf, absf);
175 __m128i b_isregular = _mm_cmpgt_epi32(c_f16max, absf_int);
176 __m128i nan_payload = _mm_and_si128(_mm_srli_epi32(absf_int, 13),
178 __m128i nan_quiet = _mm_or_si128(nan_payload, c_nanbit);
179 __m128i nanfinal = _mm_and_si128(_mm_castps_si128(b_isnan), nan_quiet);
180 __m128i inf_or_nan = _mm_or_si128(nanfinal, c_infty_as_fp16);
183 __m128i b_issub = _mm_cmpgt_epi32(c_min_normal, absf_int);
186 __m128 subnorm1 = _mm_add_ps(
187 absf, _mm_castsi128_ps(c_subnorm_magic));
188 __m128i subnorm2 = _mm_sub_epi32(_mm_castps_si128(subnorm1),
192 __m128i mantoddbit = _mm_slli_epi32(absf_int, 31 - 13);
193 __m128i mantodd = _mm_srai_epi32(mantoddbit, 31);
195 __m128i round1 = _mm_add_epi32(absf_int, c_normal_bias);
197 __m128i round2 = _mm_sub_epi32(round1, mantodd);
198 __m128i normal = _mm_srli_epi32(round2, 13);
201 __m128i nonspecial = _mm_or_si128(_mm_and_si128(subnorm2, b_issub),
202 _mm_andnot_si128(b_issub, normal));
205 __m128i joined = _mm_or_si128(_mm_and_si128(nonspecial, b_isregular),
206 _mm_andnot_si128(b_isregular, inf_or_nan));
208 __m128i sign_shift = _mm_srai_epi32(_mm_castps_si128(justsign), 16);
209 __m128i
result = _mm_or_si128(joined, sign_shift);
215static inline __m128 F16_to_F32_4x(
const __m128i &h)
217 const __m128i mask_nosign = _mm_set1_epi32(0x7fff);
224 const __m128i min_normal = _mm_set1_epi32(1 << 23);
225 const __m128i normal_bias = _mm_set1_epi32((254 - 15 - 127) << 23);
226 const __m128i subnorm_bias = _mm_set1_epi32((254 - 15 - 126) << 23);
227 const __m128i was_infnan = _mm_set1_epi32(0x7bff);
228 const __m128 exp_infnan = _mm_castsi128_ps(_mm_set1_epi32(255 << 23));
229 const __m128i was_nan = _mm_set1_epi32(0x7c00);
230 const __m128i nan_quiet = _mm_set1_epi32(1 << 22);
232 __m128i expmant = _mm_and_si128(mask_nosign, h);
233 __m128i justsign = _mm_xor_si128(h, expmant);
234 __m128i shifted = _mm_slli_epi32(expmant, 13);
235 __m128i b_wassubnorm = _mm_cmpgt_epi32(min_normal, shifted);
236 __m128 scaled_normal = _mm_castsi128_ps(_mm_add_epi32(shifted, normal_bias));
237 __m128 scaled_subnorm = _mm_sub_ps(_mm_castsi128_ps(_mm_add_epi32(shifted, subnorm_bias)),
238 _mm_castsi128_ps(subnorm_bias));
239 __m128
scaled = _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(b_wassubnorm), scaled_subnorm),
240 _mm_andnot_ps(_mm_castsi128_ps(b_wassubnorm), scaled_normal));
241 __m128i b_wasinfnan = _mm_cmpgt_epi32(expmant, was_infnan);
242 __m128i
sign = _mm_slli_epi32(justsign, 16);
243 __m128 infnanexp = _mm_and_ps(_mm_castsi128_ps(b_wasinfnan), exp_infnan);
244 __m128i b_wasnan = _mm_cmpgt_epi32(expmant, was_nan);
245 __m128i nanquiet = _mm_and_si128(b_wasnan, nan_quiet);
246 __m128 infnandone = _mm_or_ps(infnanexp, _mm_castsi128_ps(nanquiet));
248 __m128 sign_inf = _mm_or_ps(_mm_castsi128_ps(
sign), infnandone);
259#if defined(USE_HARDWARE_FP16_F16C)
260 for (; i + 7 <
length; i += 8) {
261 __m256 src8 = _mm256_loadu_ps(src);
262 __m128i h8 = _mm256_cvtps_ph(src8, _MM_FROUND_TO_NEAREST_INT);
263 _mm_storeu_si128((__m128i *)dst, h8);
267#elif defined(USE_SSE2_FP16)
268 for (; i + 3 <
length; i += 4) {
269 __m128 src4 = _mm_loadu_ps(src);
270 __m128i h4 = F32_to_F16_4x(src4);
271 __m128i h4_packed = _mm_packs_epi32(h4, h4);
272 _mm_storeu_si64(dst, h4_packed);
276#elif defined(USE_HARDWARE_FP16_NEON)
277 for (; i + 3 <
length; i += 4) {
278 float32x4_t src4 = vld1q_f32(src);
279 float16x4_t h4 = vcvt_f16_f32(src4);
280 vst1_f16((float16_t *)dst, h4);
293 const float *src,
uint16_t *dst,
size_t length,
float min_value,
float max_value)
296#if defined(USE_HARDWARE_FP16_F16C)
297 __m256 min8 = _mm256_set1_ps(min_value);
298 __m256 max8 = _mm256_set1_ps(max_value);
299 for (; i + 7 <
length; i += 8) {
300 __m256 src8 = _mm256_loadu_ps(src);
302 __m256 not_nans_mask = _mm256_cmp_ps(src8, src8, _CMP_EQ_OQ);
303 src8 = _mm256_and_ps(src8, not_nans_mask);
305 src8 = _mm256_min_ps(_mm256_max_ps(src8, min8), max8);
307 __m128i h8 = _mm256_cvtps_ph(src8, _MM_FROUND_TO_NEAREST_INT);
308 _mm_storeu_si128((__m128i *)dst, h8);
312#elif (defined(USE_SSE2_FP16) && defined(__SSE4_1__))
313 __m128
min4 = _mm_set1_ps(min_value);
314 __m128
max4 = _mm_set1_ps(max_value);
315 for (; i + 3 <
length; i += 4) {
316 __m128 src4 = _mm_loadu_ps(src);
318 __m128 not_nans_mask = _mm_cmpeq_ps(src4, src4);
319 src4 = _mm_and_ps(src4, not_nans_mask);
321 src4 = _mm_min_ps(_mm_max_ps(src4,
min4),
max4);
323 __m128i h4 = F32_to_F16_4x(src4);
324 __m128i h4_packed = _mm_packs_epi32(h4, h4);
325 _mm_storeu_si64(dst, h4_packed);
329#elif defined(USE_HARDWARE_FP16_NEON)
330 float32x4_t
min4 = vdupq_n_f32(min_value);
331 float32x4_t
max4 = vdupq_n_f32(max_value);
332 for (; i + 3 <
length; i += 4) {
333 float32x4_t src4 = vld1q_f32(src);
335 uint32x4_t not_nans_mask = vceqq_f32(src4, src4);
336 src4 = vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(src4), not_nans_mask));
337 src4 = vminq_f32(vmaxq_f32(src4,
min4),
max4);
338 float16x4_t h4 = vcvt_f16_f32(src4);
339 vst1_f16((float16_t *)dst, h4);
347 float v =
clamp(*src++, min_value, max_value);
358#if defined(USE_HARDWARE_FP16_F16C)
359 for (; i + 7 <
length; i += 8) {
360 __m256 src8 = _mm256_loadu_ps(src);
361 __m128i h8 = _mm256_cvtps_ph(src8, _MM_FROUND_TO_NEAREST_INT);
364 const __m128i exp_mask = _mm_set1_epi16(0x7c00u);
365 __m128i exp_all_ones = _mm_cmpeq_epi16(_mm_and_si128(h8, exp_mask), exp_mask);
366 const __m128i mant_mask = _mm_set1_epi16(0x03ffu);
367 const __m128i zero = _mm_setzero_si128();
368 __m128i mant_is_zero = _mm_cmpeq_epi16(_mm_and_si128(h8, mant_mask), zero);
369 __m128i is_inf = _mm_and_si128(exp_all_ones, mant_is_zero);
370 const __m128i all_ones = _mm_cmpeq_epi16(zero, zero);
371 __m128i is_nan = _mm_and_si128(exp_all_ones, _mm_andnot_si128(mant_is_zero, all_ones));
372 const __m128i sign_mask = _mm_set1_epi16(0x8000u);
373 __m128i signbits = _mm_and_si128(h8, sign_mask);
374 __m128i inf_res = _mm_or_si128(signbits, _mm_set1_epi16(0x7bffu));
375 __m128i nan_res = signbits;
377 h8 = _mm_blendv_epi8(h8, inf_res, is_inf);
378 h8 = _mm_blendv_epi8(h8, nan_res, is_nan);
380 _mm_storeu_si128((__m128i *)dst, h8);
384#elif (defined(USE_SSE2_FP16) && defined(__SSE4_1__))
385 for (; i + 3 <
length; i += 4) {
386 __m128 src4 = _mm_loadu_ps(src);
387 __m128i h4 = F32_to_F16_4x(src4);
390 __m128i hi_part = _mm_and_si128(h4, _mm_set1_epi32(0xffff0000u));
391 const __m128i exp_mask = _mm_set1_epi16(0x7c00u);
392 __m128i exp_all_ones = _mm_cmpeq_epi16(_mm_and_si128(h4, exp_mask), exp_mask);
393 const __m128i mant_mask = _mm_set1_epi16(0x03ffu);
394 const __m128i zero = _mm_setzero_si128();
395 __m128i mant_is_zero = _mm_cmpeq_epi16(_mm_and_si128(h4, mant_mask), zero);
396 __m128i is_inf = _mm_and_si128(exp_all_ones, mant_is_zero);
397 const __m128i all_ones = _mm_cmpeq_epi16(zero, zero);
398 __m128i is_nan = _mm_and_si128(exp_all_ones, _mm_andnot_si128(mant_is_zero, all_ones));
399 const __m128i sign_mask = _mm_set1_epi16(0x8000u);
400 __m128i signbits = _mm_and_si128(h4, sign_mask);
401 __m128i inf_res = _mm_or_si128(signbits, _mm_set1_epi16(0x7bffu));
402 __m128i nan_res = signbits;
404 h4 = _mm_blendv_epi8(h4, inf_res, is_inf);
405 h4 = _mm_blendv_epi8(h4, nan_res, is_nan);
406 h4 = _mm_and_si128(h4, _mm_set1_epi32(0xffff));
407 h4 = _mm_or_si128(h4, hi_part);
409 __m128i h4_packed = _mm_packs_epi32(h4, h4);
410 _mm_storeu_si64(dst, h4_packed);
414#elif defined(USE_HARDWARE_FP16_NEON)
415 for (; i + 3 <
length; i += 4) {
416 float32x4_t src4 = vld1q_f32(src);
417 float16x4_t h4 = vcvt_f16_f32(src4);
420 uint16x4_t hu4 = vreinterpret_u16_f16(h4);
421 const uint16x4_t exp_mask = vdup_n_u16(0x7c00u);
422 uint16x4_t exp_all_ones = vceq_u16(vand_u16(hu4, exp_mask), exp_mask);
423 const uint16x4_t mant_mask = vdup_n_u16(0x03ffu);
424 const uint16x4_t zero = vdup_n_u16(0);
425 uint16x4_t mant_is_zero = vceq_u16(vand_u16(hu4, mant_mask), zero);
426 uint16x4_t is_inf = vand_u16(exp_all_ones, mant_is_zero);
427 uint16x4_t is_nan = vand_u16(exp_all_ones, vmvn_u16(mant_is_zero));
428 const uint16x4_t sign_mask = vdup_n_u16(0x8000u);
429 uint16x4_t signbits = vand_u16(hu4, sign_mask);
430 uint16x4_t inf_res = vorr_u16(signbits, vdup_n_u16(0x7bffu));
431 uint16x4_t nan_res = signbits;
433 hu4 = vbsl_u16(is_inf, inf_res, hu4);
434 hu4 = vbsl_u16(is_nan, nan_res, hu4);
435 h4 = vreinterpret_f16_u16(hu4);
437 vst1_f16((float16_t *)dst, h4);
452#if defined(USE_HARDWARE_FP16_F16C)
453 for (; i + 7 <
length; i += 8) {
454 __m128i src8 = _mm_loadu_si128((__m128i *)src);
455 __m256 f8 = _mm256_cvtph_ps(src8);
456 _mm256_storeu_ps(dst, f8);
460#elif defined(USE_SSE2_FP16)
461 for (; i + 3 <
length; i += 4) {
462 __m128i src4 = _mm_loadu_si64(src);
463 src4 = _mm_unpacklo_epi16(src4, src4);
464 __m128 f4 = F16_to_F32_4x(src4);
465 _mm_storeu_ps(dst, f4);
469#elif defined(USE_HARDWARE_FP16_NEON)
470 for (; i + 3 <
length; i += 4) {
471 float16x4_t src4 = vld1_f16((
const float16_t *)src);
472 float32x4_t f4 = vcvt_f32_f16(src4);
485#ifdef USE_HARDWARE_FP16_NEON
486# undef USE_HARDWARE_FP16_NEON
488#ifdef USE_HARDWARE_FP16_F16C
489# undef USE_HARDWARE_FP16_F16C
btMatrix3x3 scaled(const btVector3 &s) const
Create a scaled copy of the matrix.
ccl_device_inline T min4(const T &a, const T &b, const T &c, const T &d)
ccl_device_inline T max4(const T &a, const T &b, const T &c, const T &d)
void float_to_half_array(const float *src, uint16_t *dst, size_t length)
uint16_t float_to_half(float v)
T clamp(const T &a, const T &min, const T &max)
T length(const VecBase< T, Size > &a)
void float_to_half_clamp_array(const float *src, uint16_t *dst, size_t length, float min_value, float max_value)
void half_to_float_array(const uint16_t *src, float *dst, size_t length)
void float_to_half_make_finite_array(const float *src, uint16_t *dst, size_t length)
float half_to_float(uint16_t v)
uint16_t float_to_half_make_finite(float v)
ATTR_WARN_UNUSED_RESULT const BMVert * v