Blender V5.3
BLI_simd.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
5#pragma once
6
12
13#if (defined(__ARM_NEON) || (defined(_M_ARM64) && defined(_MSC_VER))) && \
14 defined(WITH_SSE2NEON) && !defined(DISABLE_SSE2NEON)
15/* SSE/SSE2 emulation on ARM Neon. Match SSE precision. */
16# if !defined(SSE2NEON_PRECISE_MINMAX)
17# define SSE2NEON_PRECISE_MINMAX 1
18# endif
19# if !defined(SSE2NEON_PRECISE_DIV)
20# define SSE2NEON_PRECISE_DIV 1
21# endif
22# if !defined(SSE2NEON_PRECISE_SQRT)
23# define SSE2NEON_PRECISE_SQRT 1
24# endif
25# include <sse2neon.h>
26# define BLI_HAVE_SSE2 1
27# define BLI_HAVE_ARM_NEON 1
28#elif defined(__SSE2__)
29/* Native SSE2 on Intel/AMD. */
30# include <emmintrin.h>
31# define BLI_HAVE_SSE2 1
32# define BLI_HAVE_ARM_NEON 0
33#else
34# define BLI_HAVE_SSE2 0
35# define BLI_HAVE_ARM_NEON 0
36#endif
37
38#if (defined(__ARM_NEON) || (defined(_M_ARM64) && defined(_MSC_VER))) && \
39 defined(WITH_SSE2NEON) && !defined(DISABLE_SSE2NEON)
40/* SSE4 is emulated via sse2neon. */
41# define BLI_HAVE_SSE4 1
42#elif defined(__SSE4_2__)
43/* Native SSE4.2. */
44# include <nmmintrin.h>
45# define BLI_HAVE_SSE4 1
46#else
47# define BLI_HAVE_SSE4 0
48#endif
49
50/* Similar to Cycles simd.h. */
51
52#include <cstdint>
53
54#if defined(FREE_WINDOWS64)
55# include <windows.h>
56#elif defined(_MSC_VER) && !defined(__KERNEL_NEON__)
57# include <intrin.h>
58#elif (defined(__x86_64__) || defined(__i386__))
59# include <x86intrin.h>
60#elif defined(__KERNEL_NEON__)
61# define SSE2NEON_PRECISE_MINMAX 1
62# include <sse2neon.h>
63#endif
64
65/* Floating Point Control, for Embree. */
66#if defined(__x86_64__) || defined(_M_X64)
67# define SIMD_SET_FLUSH_TO_ZERO \
68 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
69 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
70#elif defined(__aarch64__) || defined(_M_ARM64)
71/* The get/set denormals to zero was implemented in sse2neon v1.5.0.
72 * Keep the compatibility code until the minimum library version is increased. */
73# if defined(_MM_SET_FLUSH_ZERO_MODE)
74# define SIMD_SET_FLUSH_TO_ZERO \
75 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); \
76 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
77# elif !defined(_M_ARM64)
78# define _MM_FLUSH_ZERO_ON 24
79# define __get_fpcr(__fpcr) __asm__ __volatile__("mrs %0,fpcr" : "=r"(__fpcr))
80# define __set_fpcr(__fpcr) __asm__ __volatile__("msr fpcr,%0" : : "ri"(__fpcr))
81# define SIMD_SET_FLUSH_TO_ZERO blender::set_fz(_MM_FLUSH_ZERO_ON);
82# define SIMD_GET_FLUSH_TO_ZERO blender::get_fz(_MM_FLUSH_ZERO_ON)
83# else
84# define _MM_FLUSH_ZERO_ON 24
85# define __get_fpcr(__fpcr) __fpcr = _ReadStatusReg(0x5A20)
86# define __set_fpcr(__fpcr) _WriteStatusReg(0x5A20, __fpcr)
87# define SIMD_SET_FLUSH_TO_ZERO blender::set_fz(_MM_FLUSH_ZERO_ON);
88# define SIMD_GET_FLUSH_TO_ZERO blender::get_fz(_MM_FLUSH_ZERO_ON)
89# endif
90#else
91# define SIMD_SET_FLUSH_TO_ZERO
92#endif
93
94namespace blender {
95
96#if (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_MM_SET_FLUSH_ZERO_MODE)
97inline int set_fz(const uint32_t flag)
98{
99 uint64_t old_fpcr;
100 uint64_t new_fpcr;
101 __get_fpcr(old_fpcr);
102 new_fpcr = old_fpcr | (uint64_t(1) << flag);
103 __set_fpcr(new_fpcr);
104 __get_fpcr(old_fpcr);
105 return old_fpcr == new_fpcr;
106}
107inline int get_fz(const uint32_t flag)
108{
109 uint64_t cur_fpcr;
110 __get_fpcr(cur_fpcr);
111 return (cur_fpcr & (uint64_t(1) << flag)) > 0 ? 1 : 0;
112}
113#endif
114
124/* Same platforms that use the `_mm_*` intrinsics for #SIMD_SET_FLUSH_TO_ZERO above. */
125#if defined(__x86_64__) || defined(_M_X64) || \
126 ((defined(__aarch64__) || defined(_M_ARM64)) && defined(_MM_SET_FLUSH_ZERO_MODE))
127 unsigned int orig_csr_;
128
129 public:
130 ScopedFlushToZero() : orig_csr_(_mm_getcsr())
131 {
133 }
134
136 {
137 _mm_setcsr(orig_csr_);
138 }
139
140#endif
141};
142
148#if defined(__x86_64__) || defined(_M_X64)
149 unsigned int orig_csr_;
150
151 public:
152 ScopedNoFlushToZero() : orig_csr_(_mm_getcsr())
153 {
154 /* MXCSR: bit 15 is flush-to-zero, bit 6 is denormals-are-zero. */
155 _mm_setcsr(orig_csr_ & ~((1u << 15) | (1u << 6)));
156 }
157
159 {
160 _mm_setcsr(orig_csr_);
161 }
162#elif (defined(__aarch64__) || defined(_M_ARM64)) && defined(_MM_SET_FLUSH_ZERO_MODE)
163 /* On ARM both modes share the same FPCR bit. Save and restore both for symmetry with
164 * #SIMD_SET_FLUSH_TO_ZERO. */
165 unsigned int orig_fz_;
166 unsigned int orig_daz_;
167
168 public:
170 : orig_fz_(_MM_GET_FLUSH_ZERO_MODE()), orig_daz_(_MM_GET_DENORMALS_ZERO_MODE())
171 {
172 _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_OFF);
173 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_OFF);
174 }
175
177 {
178 _MM_SET_FLUSH_ZERO_MODE(orig_fz_);
179 _MM_SET_DENORMALS_ZERO_MODE(orig_daz_);
180 }
181#elif defined(__aarch64__) || defined(_M_ARM64)
182 uint64_t orig_fpcr_;
183
184 public:
186 {
187 __get_fpcr(orig_fpcr_);
188 __set_fpcr(orig_fpcr_ & ~(uint64_t(1) << _MM_FLUSH_ZERO_ON));
189 }
190
192 {
193 __set_fpcr(orig_fpcr_);
194 }
195#else
196 public:
198#endif
199};
200
201} // namespace blender
#define SIMD_SET_FLUSH_TO_ZERO
Definition BLI_simd.hh:91
unsigned long long int uint64_t
BMesh Iterators.
unsigned __int64 uint64_t
Definition BKE_text.h:91
unsigned int uint32_t
Definition BKE_text.h:81