16#if !defined(__KERNEL_GPU__) && defined(__KERNEL_SSE2__)
25#if !defined(__KERNEL_GPU__)
32 half(
const uint16_t &i) : v(i) {}
33 operator uint16_t()
const
48#if !defined(__KERNEL_METAL__)
58static_assert(
sizeof(
half) == 2);
60static_assert(
sizeof(
half4) == 8);
62#if !defined(__KERNEL_GPU__)
83 const int c_min_normal = 1 << 23;
84 const int c_normal_bias = 112 << 23;
85 const int c_denorm_bias = 113 << 23;
87 const uint32_t bits = uint16_t(h);
88 const uint32_t s = (bits & 0x8000) << 16;
89 const int em = int((bits & 0x7fff) << 13);
93 const float f = (em < c_min_normal) ? denorm : normal;
104 const int4 i =
make_int4(uint16_t(h.
x), uint16_t(h.
y), uint16_t(h.
z), uint16_t(h.
w));
105 const int4 s = (i & 0x8000) << 16;
106 const int4 em = (i & 0x7fff) << 13;
108 const int4 b_isdenorm = c_min_normal > em;
110 const float4 normal =
cast(em + c_normal_bias);
111 const float4 denorm =
cast(em + c_denorm_bias) -
cast(c_denorm_bias);
124 const int c_f16max = (127 + 16) << 23;
125 const int c_infty_as_fp16 = 0x7c00;
126 const int c_min_normal = (127 - 14) << 23;
127 const int c_denorm_magic = ((127 - 15) + (23 - 10) + 1) << 23;
128 const int c_normal_bias = 0xfff - ((127 - 15) << 23);
131 const uint sign_i = f_i & 0x80000000u;
132 const int abs_i = int(f_i ^ sign_i);
136 if (abs_i >= c_f16max) {
138 res = uint16_t(c_infty_as_fp16);
140 else if (abs_i < c_min_normal) {
145 res = uint16_t(denorm_i);
149 const int mant_odd = int(uint(abs_i) >> 13) & 1;
150 res = uint16_t((abs_i + c_normal_bias + mant_odd) >> 13);
153 return half(res | uint16_t(sign_i >> 16));
161 const int4 c_denorm_magic =
make_int4(((127 - 15) + (23 - 10) + 1) << 23);
162 const int4 c_normal_bias =
make_int4(0xfff - ((127 - 15) << 23));
167 const int4 b_isregular = c_f16max > abs_i;
168 const int4 b_isdenorm = c_min_normal > abs_i;
175 const int4 mant_odd = (abs_i << (31 - 13)) >> 31;
176 const int4 normal =
srl(abs_i + c_normal_bias - mant_odd, 13);
179 const int4 nonspecial =
select(b_isdenorm, denorm_i, normal);
182 const int4 combined =
select(b_isregular, nonspecial, c_infty_as_fp16);
185 const int4 res = combined | (sign_i >> 16);
194 return {h.
x, h.
y, h.
z};
200#if defined(__KERNEL_METAL__) || defined(__KERNEL_ONEAPI__)
202#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
207#elif defined(__ARM_NEON) || defined(_M_ARM64)
208 uint16x4_t v = vdup_n_u16(uint16_t(h));
209 return vgetq_lane_f32(vcvt_f32_f16(vreinterpret_f16_u16(v)), 0);
210#elif defined(__F16C__)
211 return _cvtsh_ss(uint16_t(h));
220#if defined(__KERNEL_METAL__) || defined(__KERNEL_ONEAPI__)
222#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
224#elif defined(__ARM_NEON) || defined(_M_ARM64)
225 return half(vget_lane_u16(vreinterpret_u16_f16(vcvt_f16_f32(vdupq_n_f32(f))), 0));
226#elif defined(__F16C__)
227 return half((uint16_t)_cvtss_sh(f, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));
235#if defined(__KERNEL_METAL__)
237#elif defined(__KERNEL_ONEAPI__)
239#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
241#elif defined(__ARM_NEON) || defined(_M_ARM64)
243 vst1_u16(
reinterpret_cast<uint16_t *
>(&r),
244 vreinterpret_u16_f16(vcvt_f16_f32(float32x4_t{f.
x, f.
y, f.
z, f.
w})));
246#elif defined(__KERNEL_SSE__) && defined(__F16C__)
248 const __m128i h = _mm_cvtps_ph(_mm_loadu_ps(&f.
x),
249 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
250 _mm_storel_epi64(
reinterpret_cast<__m128i *
>(&r), h);
259#if defined(__KERNEL_METAL__)
261#elif defined(__KERNEL_ONEAPI__)
263#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
265#elif defined(__ARM_NEON) || defined(_M_ARM64)
268 vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(
reinterpret_cast<const uint16_t *
>(&h)))));
270#elif defined(__KERNEL_SSE__) && defined(__F16C__)
272 _mm_storeu_ps(&r.
x, _mm_cvtph_ps(_mm_loadl_epi64(
reinterpret_cast<const __m128i *
>(&h))));
281#if defined(__KERNEL_GPU__)
283#elif defined(__ARM_NEON) || defined(_M_ARM64)
284 const uint16x4_t h = vreinterpret_u16_f16(vcvt_f16_f32(float32x4_t{f.
x, f.
y, f.
z, 0.0f}));
285 return {
half(vget_lane_u16(h, 0)),
half(vget_lane_u16(h, 1)),
half(vget_lane_u16(h, 2))};
286#elif defined(__KERNEL_SSE__) && defined(__F16C__)
287 const __m128i h = _mm_cvtps_ph(_mm_set_ps(0.0f, f.
z, f.
y, f.
x),
288 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
289 return {
half(uint16_t(_mm_extract_epi16(h, 0))),
290 half(uint16_t(_mm_extract_epi16(h, 1))),
291 half(uint16_t(_mm_extract_epi16(h, 2)))};
294 return {h.x, h.y, h.z};
300#if defined(__KERNEL_GPU__)
302#elif defined(__ARM_NEON) || defined(_M_ARM64)
303 const float32x4_t f = vcvt_f32_f16(
304 vreinterpret_f16_u16(uint16x4_t{uint16_t(h.
x), uint16_t(h.
y), uint16_t(h.
z), 0}));
305 return make_float3(vgetq_lane_f32(f, 0), vgetq_lane_f32(f, 1), vgetq_lane_f32(f, 2));
306#elif defined(__KERNEL_SSE__) && defined(__F16C__)
307 const __m128i v = _mm_set_epi16(0, 0, 0, 0, 0, uint16_t(h.
z), uint16_t(h.
y), uint16_t(h.
x));
308 const __m128 f = _mm_cvtph_ps(v);
342#ifndef __KERNEL_GPU__
345 const int exponent = (uint16_t(h) >> 10) & 0x001f;
346 return exponent < 31;
half & operator=(const uint16_t &i)
#define ccl_device_inline
ccl_device_forceinline float __half2float(const half h)
#define CCL_NAMESPACE_END
constexpr T clamp(T, U, U) RET
ccl_device_inline float half_is_finite(const half h)
ccl_device_inline half float_to_half_display(const float f)
ccl_device_inline float4 fallback_half4_to_float4(const half4 h)
ccl_device_inline float3 fallback_packed_half3_to_float3(const packed_half3 h)
ccl_device_inline half float_to_half_image(const float f)
ccl_device_inline float3 packed_half3_to_float3(const packed_half3 h)
ccl_device_inline half4 fallback_float4_to_half4(const float4 f)
ccl_device_inline packed_half3 fallback_float3_to_packed_half3(const float3 f)
ccl_device_inline float4 half4_to_float4_image(const half4 h)
ccl_device_inline float half_to_float_image(half h)
ccl_device_inline half4 float4_to_half4(const float4 f)
ccl_device_inline float half_to_float(half h)
ccl_device_inline packed_half3 float3_to_packed_half3(const float3 f)
ccl_device_inline float4 half4_to_float4(const half4 h)
ccl_device_inline half fallback_float_to_half(const float f)
ccl_device_inline half4 float4_to_half4_display(const float4 f)
ccl_device_inline float fallback_half_to_float(const half h)
ccl_device_inline half float_to_half(const float f)
ccl_device_inline dual3 fabs(const ccl_private dual3 &a)
ccl_device_inline int4 __float4_as_int4(const float4 f)
ccl_device_inline int4 cast(const float4 a)
ccl_device_inline float4 __int4_as_float4(const int4 i)
ccl_device_inline int4 srl(const int4 a, const int i)
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float