Blender V5.3
gl_compilation_subprocess.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2024 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
10
11#if BLI_SUBPROCESS_SUPPORT
12
13# include "BKE_appdir.hh"
14# include "BLI_fileops.hh"
15# include "BLI_hash.hh"
16# include "BLI_path_utils.hh"
17# include "BLI_string.hh"
18# include "BLI_threads.hh"
19# include "CLG_log.h"
20# include "GHOST_IContext.hh"
21# include "GHOST_ISystem.hh"
22# include "GPU_context.hh"
23# include "GPU_init_exit.hh"
25# include <iostream>
26# include <string>
27
28# ifndef _WIN32
29# include <unistd.h>
30# else
31# include "BLI_winstuff.hh"
32# endif
33
34/* Include after `BLI_winstuff.hh` to avoid APIENTRY redefinition. */
35# include <epoxy/gl.h>
36
37namespace blender {
38
39namespace gpu {
40
41class SubprocessShader {
42 GLuint comp_ = 0;
43 GLuint vert_ = 0;
44 GLuint geom_ = 0;
45 GLuint frag_ = 0;
46 GLuint program_ = 0;
47 bool success_ = false;
48
49 public:
50 SubprocessShader(const char *comp_src,
51 const char *vert_src,
52 const char *geom_src,
53 const char *frag_src)
54 {
55 GLint status;
56 program_ = glCreateProgram();
57
58 auto compile_stage = [&](const char *src, GLenum stage) -> GLuint {
59 if (src == nullptr) {
60 /* We only want status errors if compilation fails. */
61 status = GL_TRUE;
62 return 0;
63 }
64
65 GLuint shader = glCreateShader(stage);
66 glShaderSource(shader, 1, &src, nullptr);
67 glCompileShader(shader);
68 glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
69 glAttachShader(program_, shader);
70 return shader;
71 };
72
73 comp_ = compile_stage(comp_src, GL_COMPUTE_SHADER);
74 if (!status) {
75 return;
76 }
77
78 vert_ = compile_stage(vert_src, GL_VERTEX_SHADER);
79 if (!status) {
80 return;
81 }
82
83 geom_ = compile_stage(geom_src, GL_GEOMETRY_SHADER);
84 if (!status) {
85 return;
86 }
87
88 frag_ = compile_stage(frag_src, GL_FRAGMENT_SHADER);
89 if (!status) {
90 return;
91 }
92
93 glLinkProgram(program_);
94 glGetProgramiv(program_, GL_LINK_STATUS, &status);
95 if (!status) {
96 return;
97 }
98
99 success_ = true;
100 }
101
102 ~SubprocessShader()
103 {
104 glDeleteShader(comp_);
105 glDeleteShader(vert_);
106 glDeleteShader(geom_);
107 glDeleteShader(frag_);
108 glDeleteProgram(program_);
109 }
110
111 ShaderBinaryHeader *get_binary(void *memory)
112 {
113 ShaderBinaryHeader *bin = reinterpret_cast<ShaderBinaryHeader *>(memory);
114 bin->format = 0;
115 bin->size = 0;
116
117 if (success_) {
118 glGetProgramiv(program_, GL_PROGRAM_BINARY_LENGTH, &bin->size);
119 if (bin->size > sizeof(ShaderBinaryHeader::data)) {
120 bin->size = 0;
121 return nullptr;
122 }
123 glGetProgramBinary(program_, bin->size, nullptr, &bin->format, bin->data);
124 }
125
126 return bin;
127 }
128};
129
130/* Check if the binary is valid and can be loaded by the driver. */
131static bool validate_binary(void *binary)
132{
133 ShaderBinaryHeader *bin = reinterpret_cast<ShaderBinaryHeader *>(binary);
134 GLuint program = glCreateProgram();
135 glProgramBinary(program, bin->format, bin->data, bin->size);
136 GLint status;
137 glGetProgramiv(program, GL_LINK_STATUS, &status);
138 glDeleteProgram(program);
139 return status;
140}
141
142std::string GL_shader_cache_dir_get()
143{
144 static char tmp_dir_buffer[1024];
145 BKE_appdir_folder_caches(tmp_dir_buffer, sizeof(tmp_dir_buffer));
146
147 std::string cache_dir = std::string(tmp_dir_buffer) + "gl-shader-cache" + SEP_STR;
148 BLI_dir_create_recursive(cache_dir.c_str());
149
150 return cache_dir;
151}
152
153} // namespace gpu
154
155void GPU_compilation_subprocess_run(const char *subprocess_name)
156{
157 using namespace blender::gpu;
158
159# ifndef _WIN32
161 pid_t ppid = getppid();
162# endif
163
164 CLG_init();
166
167 /* Prevent the ShaderCompiler from spawning extra threads/contexts, we don't need them. */
169
170 std::string name = subprocess_name;
171 SharedMemory shared_mem(name, compilation_subprocess_shared_memory_size, false);
172 if (!shared_mem.get_data()) {
173 std::cerr << "Compilation Subprocess: Failed to open shared memory " << subprocess_name
174 << "\n";
175 return;
176 }
177 SharedSemaphore start_semaphore(name + "_START", true);
178 SharedSemaphore end_semaphore(name + "_END", true);
179 SharedSemaphore close_semaphore(name + "_CLOSE", true);
180
182 GHOST_ISystem *ghost_system = GHOST_ISystem::getSystem();
183 BLI_assert(ghost_system);
184 GPU_backend_ghost_system_set(ghost_system);
185 GHOST_GPUSettings gpu_settings = {0};
186 gpu_settings.context_type = GHOST_kDrawingContextTypeOpenGL;
187 GHOST_IContext *ghost_context = ghost_system->createOffscreenContext(gpu_settings);
188 if (ghost_context == nullptr) {
189 std::cerr << "Compilation Subprocess: Failed to initialize GHOST context for "
190 << subprocess_name << "\n";
192 return;
193 }
194 ghost_context->activateDrawingContext();
195 GPUContext *gpu_context = GPU_context_create(nullptr, ghost_context);
196 GPU_init();
197
198 std::string cache_dir = GL_shader_cache_dir_get();
199
200 while (true) {
201 /* Process events to avoid crashes on Wayland.
202 * See https://bugreports.qt.io/browse/QTBUG-81504 */
203 ghost_system->processEvents(false);
204
205# ifdef _WIN32
206 start_semaphore.decrement();
207# else
208 bool lost_parent = false;
209 while (!lost_parent && !start_semaphore.try_decrement(1000)) {
210 lost_parent = getppid() != ppid;
211 }
212 if (lost_parent) {
213 std::cerr << "Compilation Subprocess: Lost parent process\n";
214 break;
215 }
216# endif
217
218 if (close_semaphore.try_decrement()) {
219 break;
220 }
221
222 ShaderSourceHeader *source = reinterpret_cast<ShaderSourceHeader *>(shared_mem.get_data());
223 const char *next_src = source->sources;
224 const char *comp_src = nullptr;
225 const char *vert_src = nullptr;
226 const char *geom_src = nullptr;
227 const char *frag_src = nullptr;
228
230 std::string hash_str = "_";
231
232 auto get_src = [&]() {
233 const char *src = next_src;
234 next_src += strlen(src) + sizeof('\0');
235 hash_str += std::to_string(hasher(src)) + "_";
236 return src;
237 };
238
239 if (source->type == ShaderSourceHeader::Type::COMPUTE) {
240 comp_src = get_src();
241 }
242 else {
243 vert_src = get_src();
244 if (source->type == ShaderSourceHeader::Type::GRAPHICS_WITH_GEOMETRY_STAGE) {
245 geom_src = get_src();
246 }
247 frag_src = get_src();
248 }
249
250 std::string cache_path = cache_dir + SEP_STR + hash_str;
251
252 /* TODO: This should lock the files? */
253 if (BLI_exists(cache_path.c_str())) {
254 {
255 /* Store the source hash in the shared memory.
256 * If the subprocess crashes, the main process will delete the cache file. */
257 std::string source_hash = "SOURCE_HASH:" + hash_str;
258 BLI_strncpy(reinterpret_cast<char *>(shared_mem.get_data()),
259 source_hash.c_str(),
260 source_hash.size() + 1);
261 }
262 /* Prevent old cache files from being deleted if they're still being used. */
263 BLI_file_touch(cache_path.c_str());
264 /* Read cached binary. */
265 fstream file(cache_path, std::ios::binary | std::ios::in | std::ios::ate);
266 std::streamsize size = file.tellg();
267 if (size <= compilation_subprocess_shared_memory_size) {
268 file.seekg(0, std::ios::beg);
269 /* Use temp memory so we don't overwrite the source hash. */
270 static char tmp_mem[compilation_subprocess_shared_memory_size];
271 file.read(tmp_mem, size);
272 /* Close first in case validation hangs the driver. */
273 file.close();
274 /* Ensure it's valid. */
275 if (!validate_binary(tmp_mem)) {
276 std::cout << "Compilation Subprocess: Failed to load cached shader binary " << hash_str
277 << "\n";
278 /* TODO: No longer true. */
279 /* We can't compile the shader anymore since we have written over the source code,
280 * but we delete the cache for the next time this shader is requested. */
281 BLI_delete(cache_path.c_str(), false, false);
282 }
283 /* Copy the temp memory to the shared memory now that we know loading the shader doesn't
284 * crash the driver. */
285 memcpy(shared_mem.get_data(), tmp_mem, size);
286 end_semaphore.increment();
287 continue;
288 }
289 else {
290 /* This should never happen, since shaders larger than the pool size should be discarded
291 * and compiled in the main Blender process. */
292 std::cerr << "Compilation Subprocess: Wrong size for cached shader binary " << hash_str
293 << "\n";
295 }
296 }
297
298 SubprocessShader shader(comp_src, vert_src, geom_src, frag_src);
299 ShaderBinaryHeader *binary = shader.get_binary(shared_mem.get_data());
300
301 if (binary) {
302 fstream file(cache_path, std::ios::binary | std::ios::out);
303 file.write(reinterpret_cast<char *>(shared_mem.get_data()),
304 binary->size + offsetof(ShaderBinaryHeader, data));
305 }
306
307 end_semaphore.increment();
308 }
309
310 GPU_exit();
311 GPU_context_discard(gpu_context);
312 ghost_system->disposeContext(ghost_context);
314}
315
316namespace gpu {
317void GL_shader_cache_dir_clear_old()
318{
319 std::string cache_dir = GL_shader_cache_dir_get();
320
321 direntry *entries = nullptr;
322 uint32_t dir_len = BLI_filelist_dir_contents(cache_dir.c_str(), &entries);
323 for (int i : IndexRange(dir_len)) {
324 direntry entry = entries[i];
325 if (S_ISDIR(entry.s.st_mode)) {
326 continue;
327 }
328 const time_t ts_now = time(nullptr);
329 const time_t delete_threshold = 60 /*seconds*/ * 60 /*minutes*/ * 24 /*hours*/ * 30 /*days*/;
330 if (entry.s.st_mtime + delete_threshold < ts_now) {
331 BLI_delete(entry.path, false, false);
332 }
333 }
334 BLI_filelist_free(entries, dir_len);
335}
336} // namespace gpu
337} // namespace blender
338
339#endif
File and directory operations.
Compatibility-like things for windows.
#define S_ISDIR(x)
void CLG_init()
Definition clog.cc:870
#define BLI_assert_unreachable()
virtual GHOST_TSuccess activateDrawingContext()=0
static GHOST_ISystem * getSystem()
virtual GHOST_IContext * createOffscreenContext(GHOST_GPUSettings gpu_settings)=0
virtual GHOST_TSuccess disposeContext(GHOST_IContext *context)=0
static GHOST_TSuccess disposeSystem()
virtual bool processEvents(bool waitForEvent)=0
static GHOST_TSuccess createSystemBackground()
constexpr IndexRange()=default
#define offsetof(t, d)
std::string time(const std::tm &date_time, TimeFormat format=TimeFormat::H24)
int size(const AssetLibraryReference *library_reference)
GPUCapabilities GCaps
BMesh Iterators.
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
void GPU_exit()
unsigned int BLI_filelist_dir_contents(const char *dirname, struct direntry **r_filelist)
GPUContext * GPU_context_create(GHOST_IWindow *ghost_window, GHOST_IContext *ghost_context)
bool BLI_dir_create_recursive(const char *dirname) ATTR_NONNULL()
Definition fileops_c.cc:416
bool BLI_exists(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
Definition storage.cc:404
void BLI_threadapi_init()
Definition threads.cc:116
unsigned int uint32_t
Definition BKE_text.h:81
char * BLI_strncpy(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
void GPU_init()
BMesh const char void * data
int BLI_delete(const char *path, bool dir, bool recursive) ATTR_NONNULL()
bool BLI_file_touch(const char *filepath) ATTR_NONNULL(1)
Definition fileops_c.cc:318
void BLI_filelist_free(struct direntry *filelist, unsigned int nrentries)
void BKE_appdir_folder_caches(char *path, size_t path_maxncpy) ATTR_NONNULL(1)
Definition appdir.cc:206
void GPU_backend_ghost_system_set(GHOST_ISystem *ghost_system_handle)
void GPU_context_discard(GPUContext *)
GHOST_TDrawingContextType context_type
#define SEP_STR
Definition unit.cc:41