Blender V5.3
half.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2026 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#pragma once
6
7#include "util/defines.h"
8#include "util/math_base.h"
9#include "util/math_float4.h"
10#include "util/math_int4.h"
11#include "util/types_base.h"
12#include "util/types_float4.h"
13#include "util/types_int4.h"
14#include "util/types_uint4.h"
15
16#if !defined(__KERNEL_GPU__) && defined(__KERNEL_SSE2__)
17# include "util/optimization.h" // IWYU pragma: keep
18# include "util/simd.h" // IWYU pragma: keep
19#endif
20
22
23/* Half Floats */
24
25#if !defined(__KERNEL_GPU__)
26/* GPUs have native support for this type.
27 * Implementing this as a class rather than a typedef so that the compiler can tell it apart from
28 * uint16_ts. */
29class half {
30 public:
31 half() = default;
32 half(const uint16_t &i) : v(i) {}
33 operator uint16_t() const
34 {
35 return v;
36 }
37 half &operator=(const uint16_t &i)
38 {
39 v = i;
40 return *this;
41 }
42
43 private:
44 uint16_t v;
45};
46#endif
47
48#if !defined(__KERNEL_METAL__)
50 half x, y, z;
51};
52
53struct half4 {
54 half x, y, z, w;
55};
56#endif
57
58static_assert(sizeof(half) == 2);
59static_assert(sizeof(packed_half3) == 6);
60static_assert(sizeof(half4) == 8);
61
62#if !defined(__KERNEL_GPU__)
63/* Optimized fallback implementations with fast path for normal and denormal numbers, assuming
64 * no Infs or NaNs. Based on public domain functions from.
65 *
66 * https://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/
67 * https://gist.github.com/rygorous/2144712
68 * https://gist.github.com/rygorous/2156668
69 * https://gist.github.com/rygorous/4d9e9e88cab13c703773dc767a23575f
70 *
71 * The half exponent and mantissa, shifted into float position, need to be scaled by 2^112 to get
72 * the matching float exponent. This is done by adding to the exponent field rather than by
73 * multiplying with 2^112, so that no denormal float is used as intermediate value. Such a value
74 * would be flushed to zero when the CPU runs with flush-to-zero and denormals-are-zero enabled
75 * (as it is for threads that use Embree), making every denormal half decode as zero.
76 *
77 * Denormal halves have a zero exponent, so they instead get a bias that makes them normal floats,
78 * with the implicit leading one that introduces subtracted again afterwards.
79 */
80
82{
83 const int c_min_normal = 1 << 23;
84 const int c_normal_bias = 112 << 23;
85 const int c_denorm_bias = 113 << 23;
86
87 const uint32_t bits = uint16_t(h);
88 const uint32_t s = (bits & 0x8000) << 16;
89 const int em = int((bits & 0x7fff) << 13);
90
91 const float normal = __int_as_float(em + c_normal_bias);
92 const float denorm = __int_as_float(em + c_denorm_bias) - __int_as_float(c_denorm_bias);
93 const float f = (em < c_min_normal) ? denorm : normal;
94
95 return __int_as_float(__float_as_uint(f) | s);
96}
97
99{
100 const int4 c_min_normal = make_int4(1 << 23);
101 const int4 c_normal_bias = make_int4(112 << 23);
102 const int4 c_denorm_bias = make_int4(113 << 23);
103
104 const int4 i = make_int4(uint16_t(h.x), uint16_t(h.y), uint16_t(h.z), uint16_t(h.w));
105 const int4 s = (i & 0x8000) << 16;
106 const int4 em = (i & 0x7fff) << 13;
107
108 const int4 b_isdenorm = c_min_normal > em;
109
110 const float4 normal = cast(em + c_normal_bias);
111 const float4 denorm = cast(em + c_denorm_bias) - cast(c_denorm_bias);
112 const int4 f = select(b_isdenorm, cast(denorm), cast(normal));
113
114 return cast(f | s);
115}
116
121
123{
124 const int c_f16max = (127 + 16) << 23;
125 const int c_infty_as_fp16 = 0x7c00;
126 const int c_min_normal = (127 - 14) << 23;
127 const int c_denorm_magic = ((127 - 15) + (23 - 10) + 1) << 23;
128 const int c_normal_bias = 0xfff - ((127 - 15) << 23);
129
130 const uint f_i = __float_as_uint(f);
131 const uint sign_i = f_i & 0x80000000u;
132 const int abs_i = int(f_i ^ sign_i);
133
134 uint16_t res;
135
136 if (abs_i >= c_f16max) {
137 /* Overflows to infinity. */
138 res = uint16_t(c_infty_as_fp16);
139 }
140 else if (abs_i < c_min_normal) {
141 /* Denormal. */
142 float denorm_f = __uint_as_float(uint(abs_i));
143 denorm_f += __int_as_float(c_denorm_magic);
144 const int denorm_i = int(__float_as_uint(denorm_f)) - c_denorm_magic;
145 res = uint16_t(denorm_i);
146 }
147 else {
148 /* Normal. */
149 const int mant_odd = int(uint(abs_i) >> 13) & 1;
150 res = uint16_t((abs_i + c_normal_bias + mant_odd) >> 13);
151 }
152
153 return half(res | uint16_t(sign_i >> 16));
154}
155
157{
158 const int4 c_f16max = make_int4((127 + 16) << 23);
159 const int4 c_infty_as_fp16 = make_int4(0x7c00);
160 const int4 c_min_normal = make_int4((127 - 14) << 23);
161 const int4 c_denorm_magic = make_int4(((127 - 15) + (23 - 10) + 1) << 23);
162 const int4 c_normal_bias = make_int4(0xfff - ((127 - 15) << 23));
163
164 const float4 abs_f = fabs(f);
165 const int4 abs_i = __float4_as_int4(abs_f);
166
167 const int4 b_isregular = c_f16max > abs_i;
168 const int4 b_isdenorm = c_min_normal > abs_i;
169
170 /* Denormal. */
171 const float4 denorm_f = abs_f + __int4_as_float4(c_denorm_magic);
172 const int4 denorm_i = __float4_as_int4(denorm_f) - c_denorm_magic;
173
174 /* Normal. */
175 const int4 mant_odd = (abs_i << (31 - 13)) >> 31;
176 const int4 normal = srl(abs_i + c_normal_bias - mant_odd, 13);
177
178 /* Combined normal and denormal. */
179 const int4 nonspecial = select(b_isdenorm, denorm_i, normal);
180
181 /* Combine overflow to infinity. */
182 const int4 combined = select(b_isregular, nonspecial, c_infty_as_fp16);
183
184 const int4 sign_i = __float4_as_int4(f ^ abs_f);
185 const int4 res = combined | (sign_i >> 16);
186
187 return {
188 half(uint16_t(res.x)), half(uint16_t(res.y)), half(uint16_t(res.z)), half(uint16_t(res.w))};
189}
190
196#endif
197
199{
200#if defined(__KERNEL_METAL__) || defined(__KERNEL_ONEAPI__)
201 return float(h);
202#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
203 return __half2float(h);
204/* We assume half instructions are always supported when there is ARM Neon,
205 * which implies ARMv8.2-A+. There is no official Blender minimum, but is
206 * already assumed elsewhere in Blender and not that recent. */
207#elif defined(__ARM_NEON) || defined(_M_ARM64)
208 uint16x4_t v = vdup_n_u16(uint16_t(h));
209 return vgetq_lane_f32(vcvt_f32_f16(vreinterpret_f16_u16(v)), 0);
210#elif defined(__F16C__)
211 return _cvtsh_ss(uint16_t(h));
212#else
213 /* The fallback is fast so don't bother with native instructions. */
214 return fallback_half_to_float(h);
215#endif
216}
217
219{
220#if defined(__KERNEL_METAL__) || defined(__KERNEL_ONEAPI__)
221 return half(f);
222#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
223 return __float2half(f);
224#elif defined(__ARM_NEON) || defined(_M_ARM64)
225 return half(vget_lane_u16(vreinterpret_u16_f16(vcvt_f16_f32(vdupq_n_f32(f))), 0));
226#elif defined(__F16C__)
227 return half((uint16_t)_cvtss_sh(f, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));
228#else
229 return fallback_float_to_half(f);
230#endif
231}
232
234{
235#if defined(__KERNEL_METAL__)
236 return {half(f.x), half(f.y), half(f.z), half(f.w)};
237#elif defined(__KERNEL_ONEAPI__)
238 return {half(f.x), half(f.y), half(f.z), half(f.w)};
239#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
240 return {__float2half(f.x), __float2half(f.y), __float2half(f.z), __float2half(f.w)};
241#elif defined(__ARM_NEON) || defined(_M_ARM64)
242 half4 r;
243 vst1_u16(reinterpret_cast<uint16_t *>(&r),
244 vreinterpret_u16_f16(vcvt_f16_f32(float32x4_t{f.x, f.y, f.z, f.w})));
245 return r;
246#elif defined(__KERNEL_SSE__) && defined(__F16C__)
247 half4 r;
248 const __m128i h = _mm_cvtps_ph(_mm_loadu_ps(&f.x),
249 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
250 _mm_storel_epi64(reinterpret_cast<__m128i *>(&r), h);
251 return r;
252#else
253 return fallback_float4_to_half4(f);
254#endif
255}
256
258{
259#if defined(__KERNEL_METAL__)
260 return {float(h.x), float(h.y), float(h.z), float(h.w)};
261#elif defined(__KERNEL_ONEAPI__)
262 return {float(h.x), float(h.y), float(h.z), float(h.w)};
263#elif defined(__KERNEL_CUDA__) || defined(__KERNEL_HIP__)
264 return {__half2float(h.x), __half2float(h.y), __half2float(h.z), __half2float(h.w)};
265#elif defined(__ARM_NEON) || defined(_M_ARM64)
266 float4 r;
267 vst1q_f32(&r.x,
268 vcvt_f32_f16(vreinterpret_f16_u16(vld1_u16(reinterpret_cast<const uint16_t *>(&h)))));
269 return r;
270#elif defined(__KERNEL_SSE__) && defined(__F16C__)
271 float4 r;
272 _mm_storeu_ps(&r.x, _mm_cvtph_ps(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(&h))));
273 return r;
274#else
275 return fallback_half4_to_float4(h);
276#endif
277}
278
280{
281#if defined(__KERNEL_GPU__)
282 return {float_to_half(f.x), float_to_half(f.y), float_to_half(f.z)};
283#elif defined(__ARM_NEON) || defined(_M_ARM64)
284 const uint16x4_t h = vreinterpret_u16_f16(vcvt_f16_f32(float32x4_t{f.x, f.y, f.z, 0.0f}));
285 return {half(vget_lane_u16(h, 0)), half(vget_lane_u16(h, 1)), half(vget_lane_u16(h, 2))};
286#elif defined(__KERNEL_SSE__) && defined(__F16C__)
287 const __m128i h = _mm_cvtps_ph(_mm_set_ps(0.0f, f.z, f.y, f.x),
288 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
289 return {half(uint16_t(_mm_extract_epi16(h, 0))),
290 half(uint16_t(_mm_extract_epi16(h, 1))),
291 half(uint16_t(_mm_extract_epi16(h, 2)))};
292#else
294 return {h.x, h.y, h.z};
295#endif
296}
297
299{
300#if defined(__KERNEL_GPU__)
302#elif defined(__ARM_NEON) || defined(_M_ARM64)
303 const float32x4_t f = vcvt_f32_f16(
304 vreinterpret_f16_u16(uint16x4_t{uint16_t(h.x), uint16_t(h.y), uint16_t(h.z), 0}));
305 return make_float3(vgetq_lane_f32(f, 0), vgetq_lane_f32(f, 1), vgetq_lane_f32(f, 2));
306#elif defined(__KERNEL_SSE__) && defined(__F16C__)
307 const __m128i v = _mm_set_epi16(0, 0, 0, 0, 0, uint16_t(h.z), uint16_t(h.y), uint16_t(h.x));
308 const __m128 f = _mm_cvtph_ps(v);
309 return make_float3(float4(f));
310#else
311 return make_float3(fallback_half4_to_float4({h.x, h.y, h.z, 0}));
312#endif
313}
314
315/* For image textures. */
317{
318 return float_to_half(clamp(f, -65504.0f, 65504.0f));
319}
320
322{
323 return half_to_float(h);
324}
325
330
331/* For render display. */
333{
334 return float_to_half(clamp(f, 0.0f, 65504.0f));
335}
336
341
342#ifndef __KERNEL_GPU__
344{
345 const int exponent = (uint16_t(h) >> 10) & 0x001f;
346 return exponent < 31;
347}
348#endif
349
Definition half.h:29
half(const uint16_t &i)
Definition half.h:32
half & operator=(const uint16_t &i)
Definition half.h:37
half()=default
#define ccl_device_inline
ccl_device_forceinline float __half2float(const half h)
#define CCL_NAMESPACE_END
ccl_device_forceinline float3 make_float3(const float x, const float y, const float z)
#define __float2half(x)
#define __int_as_float(x)
#define __float_as_uint(x)
ccl_device_forceinline int4 make_int4(const int x, const int y, const int z, const int w)
#define __uint_as_float(x)
#define select(A, B, C)
constexpr T clamp(T, U, U) RET
ccl_device_inline float half_is_finite(const half h)
Definition half.h:343
ccl_device_inline half float_to_half_display(const float f)
Definition half.h:332
ccl_device_inline float4 fallback_half4_to_float4(const half4 h)
Definition half.h:98
ccl_device_inline float3 fallback_packed_half3_to_float3(const packed_half3 h)
Definition half.h:117
ccl_device_inline half float_to_half_image(const float f)
Definition half.h:316
ccl_device_inline float3 packed_half3_to_float3(const packed_half3 h)
Definition half.h:298
ccl_device_inline half4 fallback_float4_to_half4(const float4 f)
Definition half.h:156
ccl_device_inline packed_half3 fallback_float3_to_packed_half3(const float3 f)
Definition half.h:191
ccl_device_inline float4 half4_to_float4_image(const half4 h)
Definition half.h:326
ccl_device_inline float half_to_float_image(half h)
Definition half.h:321
ccl_device_inline half4 float4_to_half4(const float4 f)
Definition half.h:233
ccl_device_inline float half_to_float(half h)
Definition half.h:198
ccl_device_inline packed_half3 float3_to_packed_half3(const float3 f)
Definition half.h:279
ccl_device_inline float4 half4_to_float4(const half4 h)
Definition half.h:257
ccl_device_inline half fallback_float_to_half(const float f)
Definition half.h:122
ccl_device_inline half4 float4_to_half4_display(const float4 f)
Definition half.h:337
ccl_device_inline float fallback_half_to_float(const half h)
Definition half.h:81
ccl_device_inline half float_to_half(const float f)
Definition half.h:218
ccl_device_inline dual3 fabs(const ccl_private dual3 &a)
Definition math_dual.h:241
ccl_device_inline int4 __float4_as_int4(const float4 f)
ccl_device_inline int4 cast(const float4 a)
Definition math_float4.h:32
ccl_device_inline float4 __int4_as_float4(const int4 i)
ccl_device_inline int4 srl(const int4 a, const int i)
Definition math_int4.h:205
#define make_float4
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
float z
Definition sky_math.h:136
float y
Definition sky_math.h:136
float x
Definition sky_math.h:136
float y
Definition sky_math.h:225
float z
Definition sky_math.h:225
float x
Definition sky_math.h:225
float w
Definition sky_math.h:225
Definition half.h:53
half x
Definition half.h:54
half w
Definition half.h:54
half z
Definition half.h:54
half y
Definition half.h:54
half z
Definition half.h:50
half y
Definition half.h:50
half x
Definition half.h:50