Blender V5.3
device_impl.mm
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#ifdef WITH_METAL
6
7# include <map>
8# include <mutex>
9
10# include "device/metal/device.h"
12
13# include "scene/scene.h"
14
16
17# include "util/debug.h"
18# include "util/md5.h"
19# include "util/path.h"
20# include "util/time.h"
21
22# include <TargetConditionals.h>
23# include <crt_externs.h>
24
26
27class MetalDevice;
28
29thread_mutex MetalDevice::existing_devices_mutex;
30std::map<int, MetalDevice *> MetalDevice::active_device_ids;
31
32/* Thread-safe device access for async work. Calling code must pass an appropriately scoped lock
33 * to existing_devices_mutex to safeguard against destruction of the returned instance. */
34MetalDevice *MetalDevice::get_device_by_ID(const int ID,
35 thread_scoped_lock & /*existing_devices_mutex_lock*/)
36{
37 auto it = active_device_ids.find(ID);
38 if (it != active_device_ids.end()) {
39 return it->second;
40 }
41 return nullptr;
42}
43
44bool MetalDevice::is_device_cancelled(const int ID)
45{
46 thread_scoped_lock lock(existing_devices_mutex);
47 return get_device_by_ID(ID, lock) == nullptr;
48}
49
50BVHLayoutMask MetalDevice::get_bvh_layout_mask(const uint64_t /*kernel_features*/) const
51{
52 return use_metalrt ? BVH_LAYOUT_METAL : BVH_LAYOUT_BVH2;
53}
54
55void MetalDevice::set_error(const string &error)
56{
57 static std::mutex s_error_mutex;
58 std::lock_guard<std::mutex> lock(s_error_mutex);
59
60 Device::set_error(error);
61
62 if (!has_error) {
63 LOG_ERROR << "Refer to the Cycles GPU rendering documentation for possible solutions:\n"
64 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
65 has_error = true;
66 }
67}
68
69MetalDevice::MetalDevice(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
70 : Device(info, stats, profiler, headless), image_info(this, "image_info", MEM_GLOBAL)
71{
72 @autoreleasepool {
73 {
74 /* Assign an ID for this device which we can use to query whether async shader compilation
75 * requests are still relevant. */
76 thread_scoped_lock lock(existing_devices_mutex);
77 static int existing_devices_counter = 1;
78 device_id = existing_devices_counter++;
79 active_device_ids[device_id] = this;
80 }
81
82 mtlDevId = info.num;
83
84 /* select chosen device */
85 auto usable_devices = MetalInfo::get_usable_devices();
86 assert(mtlDevId < usable_devices.size());
87 mtlDevice = usable_devices[mtlDevId];
88 metal_printf("Creating new Cycles Metal device: %s", info.description.c_str());
89
90 /* Enable increased concurrent shader compiler limit.
91 * This is also done by MTLContext::MTLContext, but only in GUI mode. */
92 if (@available(macOS 13.3, *)) {
93 [mtlDevice setShouldMaximizeConcurrentCompilation:YES];
94 }
95
96 max_threads_per_threadgroup = 512;
97
98 use_metalrt = info.use_hardware_raytracing;
99 if (const char *metalrt = getenv("CYCLES_METALRT")) {
100 use_metalrt = (atoi(metalrt) != 0);
101 }
102
103 if (const char *str = getenv("CYCLES_METALRT_EXTENDED_LIMITS")) {
104 use_metalrt_extended_limits = (atoi(str) != 0);
105 }
106
107# if defined(MAC_OS_VERSION_15_0)
108 /* Use "Ray tracing with per component motion interpolation" if available.
109 * Requires Apple9 support (https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf). */
110 if (use_metalrt && [mtlDevice supportsFamily:MTLGPUFamilyApple9]) {
111 /* Concave motion paths weren't correctly bounded prior to macOS 15.6 (#136253). */
112 if (@available(macos 15.6, *)) {
113 use_pcmi = DebugFlags().metal.use_metalrt_pcmi;
114 }
115 }
116# endif
117
118 if (getenv("CYCLES_DEBUG_METAL_CAPTURE_KERNEL")) {
119 capture_enabled = true;
120 }
121
122 /* Create a global counter sampling buffer when kernel profiling is enabled.
123 * There's a limit to the number of concurrent counter sampling buffers per device, so we
124 * create one that can be reused by successive device queues. */
125 if (auto *str = getenv("CYCLES_METAL_PROFILING")) {
126 if (atoi(str) && [mtlDevice supportsCounterSampling:MTLCounterSamplingPointAtStageBoundary])
127 {
128 NSArray<id<MTLCounterSet>> *counterSets = [mtlDevice counterSets];
129
130 NSError *error = nil;
131 MTLCounterSampleBufferDescriptor *desc = [[MTLCounterSampleBufferDescriptor alloc] init];
132 [desc setStorageMode:MTLStorageModeShared];
133 [desc setLabel:@"CounterSampleBuffer"];
134 [desc setSampleCount:MAX_SAMPLE_BUFFER_LENGTH];
135 [desc setCounterSet:counterSets[0]];
136 mtlCounterSampleBuffer = [mtlDevice newCounterSampleBufferWithDescriptor:desc
137 error:&error];
138 [mtlCounterSampleBuffer retain];
139 }
140 }
141
142 /* Set kernel_specialization_level based on user preferences. */
143 switch (info.kernel_optimization_level) {
145 kernel_specialization_level = PSO_GENERIC;
146 break;
147 default:
149 kernel_specialization_level = PSO_SPECIALIZED_INTERSECT;
150 break;
152 kernel_specialization_level = PSO_SPECIALIZED_SHADE;
153 break;
154 }
155
156 if (auto *envstr = getenv("CYCLES_METAL_SPECIALIZATION_LEVEL")) {
157 kernel_specialization_level = (MetalPipelineType)atoi(envstr);
158 }
159 metal_printf("kernel_specialization_level = %s",
160 kernel_type_as_string(
161 (MetalPipelineType)min((int)kernel_specialization_level, (int)PSO_NUM - 1)));
162
163 /* Command queue for path-tracing work on the GPU. In a situation where multiple
164 * MetalDeviceQueues are spawned from one MetalDevice, they share the same MTLCommandQueue.
165 * This is thread safe and just as performant as each having their own instance. It also
166 * adheres to best practices of maximizing the lifetime of each MTLCommandQueue. */
167 mtlComputeCommandQueue = [mtlDevice newCommandQueue];
168
169 /* Command queue for non-tracing work on the GPU. */
170 mtlGeneralCommandQueue = [mtlDevice newCommandQueue];
171
172# if defined(MAC_OS_VERSION_15_0)
173 if (@available(macos 15.0, *)) {
174 if (DebugFlags().metal.use_residency_sets_if_available) {
175 /* Use a residency set to declare all rendering resources up front, avoiding
176 * the overhead of per-encoder useResource calls on every dispatch. */
177 MTLResidencySetDescriptor *residency_set_desc = [[MTLResidencySetDescriptor alloc] init];
178 residency_set_desc.label = @"CyclesResidencySet";
179 residency_set_desc.initialCapacity = 512;
180 NSError *error = nil;
181 mtlResidencySet = [mtlDevice newResidencySetWithDescriptor:residency_set_desc
182 error:&error];
183 [residency_set_desc release];
184
185 /* Only enable residency sets if creation succeeded. Otherwise we fall back to the
186 * per-encoder useResource path. */
187 if (mtlResidencySet) {
188 mtlResidencySet_enabled = true;
189 [mtlComputeCommandQueue addResidencySet:mtlResidencySet];
190 }
191 else {
192 metal_printf("Failed to create residency set: %s",
193 [[error localizedDescription] UTF8String]);
194 }
195 }
196 }
197# endif
198
199 image_bindings = [mtlDevice newBufferWithLength:8192 options:MTLResourceStorageModeShared];
200 metal_mem_alloc(image_bindings);
201
202 launch_params_buffer = [mtlDevice newBufferWithLength:sizeof(KernelParamsMetal)
203 options:MTLResourceStorageModeShared];
204 metal_mem_alloc(launch_params_buffer);
205
206 /* Cache unified pointer so we can write kernel params directly in place. */
207 launch_params = (KernelParamsMetal *)launch_params_buffer.contents;
208 }
209}
210
211MetalDevice::~MetalDevice()
212{
213 /* Cancel any async shader compilations that are in flight. */
214 cancel();
215
216 /* This lock safeguards against destruction during use (see other uses of
217 * existing_devices_mutex). */
218 thread_scoped_lock lock(existing_devices_mutex);
219
220 /* Release textures that weren't already freed by tex_free. */
221 for (int res = 0; res < image_info.size(); res++) {
222 metal_mem_free(image_info_id_map[res]);
223 image_info_id_map[res] = nil;
224 }
225
226 /* Queue resources for release, then run flush_delayed_free_list(). */
227 free_bvh();
228 metal_mem_free(launch_params_buffer);
229 metal_mem_free(image_bindings);
230 image_info.free();
231 flush_delayed_free_list();
232
233# if defined(MAC_OS_VERSION_15_0)
234 if (@available(macos 15.0, *)) {
235 if (mtlResidencySet) {
236 [mtlResidencySet endResidency];
237 [mtlResidencySet release];
238 }
239 }
240# endif
241
242 [mtlComputeCommandQueue release];
243 [mtlGeneralCommandQueue release];
244 if (mtlCounterSampleBuffer) {
245 [mtlCounterSampleBuffer release];
246 }
247 [mtlDevice release];
248}
249
250void MetalDevice::add_to_residency_set(id<MTLResource> allocation)
251{
252# if defined(MAC_OS_VERSION_15_0)
253 if (@available(macos 15.0, *)) {
254 if (allocation && mtlResidencySet) {
255 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
256 [mtlResidencySet addAllocation:allocation];
257 mtlResidencySet_dirty = true;
258 }
259 }
260# endif
261}
262
263void MetalDevice::remove_from_residency_set(id<MTLResource> allocation)
264{
265# if defined(MAC_OS_VERSION_15_0)
266 if (@available(macos 15.0, *)) {
267 if (allocation && mtlResidencySet) {
268 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
269 [mtlResidencySet removeAllocation:allocation];
270 mtlResidencySet_dirty = true;
271 }
272 }
273# endif
274}
275
276void MetalDevice::metal_mem_alloc(id<MTLResource> allocation)
277{
278 if (allocation) {
279 stats.mem_alloc(allocation.allocatedSize);
280 add_to_residency_set(allocation);
281 }
282}
283
284void MetalDevice::metal_mem_free(id<MTLResource> allocation)
285{
286 if (allocation) {
287 stats.mem_free(allocation.allocatedSize);
288
289 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
290 /* Remove from the residency set immediately, but don't commit until next enqueue. A resource
291 * can be repurposed (e.g. a BVH refit) so a deferred removal can spuriously swap the
292 * remove-then-add to be an add-then-remove. */
293 remove_from_residency_set(allocation);
294
295 /* Defer the actual [release] until flush_delayed_free_list(), so the object stays alive for
296 * any command buffer still referencing it. */
297 delayed_free_list.push_back(allocation);
298 }
299}
300
301void MetalDevice::prepare_residency()
302{
303# if defined(MAC_OS_VERSION_15_0)
304 if (@available(macos 15.0, *)) {
305 if (mtlResidencySet) {
306 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
307 if (mtlResidencySet_dirty) {
308 mtlResidencySet_dirty = false;
309 [mtlResidencySet commit];
310 }
311 }
312 }
313# endif
314}
315
316bool MetalDevice::support_device(const uint64_t /*kernel_features*/)
317{
318 return true;
319}
320
321bool MetalDevice::check_peer_access(Device * /*peer_device*/)
322{
323 assert(0);
324 /* does peer access make sense? */
325 return false;
326}
327
328bool MetalDevice::use_adaptive_compilation()
329{
331}
332
333bool MetalDevice::use_local_atomic_sort() const
334{
336}
337
338string MetalDevice::preprocess_source(MetalPipelineType pso_type,
339 const uint64_t kernel_features,
340 string *source)
341{
342 string global_defines;
343 if (use_adaptive_compilation()) {
344 global_defines += "#define __KERNEL_FEATURES__ " + to_string(kernel_features) + "\n";
345 }
346
347 if (use_metalrt) {
348 global_defines += "#define __KERNEL_METALRT__\n";
349 if (motion_blur) {
350 global_defines += "#define __METALRT_MOTION__\n";
351 }
352 if (use_metalrt_extended_limits) {
353 global_defines += "#define __METALRT_EXTENDED_LIMITS__\n";
354 }
355 }
356
357# ifdef WITH_CYCLES_DEBUG
358 global_defines += "#define WITH_CYCLES_DEBUG\n";
359# endif
360
361 global_defines += "#define __KERNEL_METAL_APPLE__\n";
362 if (@available(macos 14.0, *)) {
363 /* Use Program Scope Global Built-ins, when available. */
364 global_defines += "#define __METAL_GLOBAL_BUILTINS__\n";
365 }
366# ifdef WITH_NANOVDB
367 /* Compiling in NanoVDB results in a marginal drop in render performance,
368 * so disable it for specialized PSOs when no images are using it. */
369 if ((pso_type == PSO_GENERIC || using_nanovdb) && DebugFlags().metal.use_nanovdb) {
370 global_defines += "#define WITH_NANOVDB\n";
371 }
372# endif
373
374# if TARGET_CPU_ARM64
375 global_defines += "#define __KERNEL_METAL_TARGET_CPU_ARM64__\n";
376# endif
377
378 /* Replace specific KernelData "dot" dereferences with a Metal function_constant identifier of
379 * the same character length. Build a string of all active constant values which is then hashed
380 * in order to identify the PSO.
381 */
382 if (pso_type != PSO_GENERIC) {
383 if (source) {
384 const double starttime = time_dt();
385
386# define KERNEL_STRUCT_BEGIN(name, parent) \
387 string_replace_same_length(*source, "kernel_data." #parent ".", "kernel_data_" #parent "_");
388
389 bool next_member_is_specialized = true;
390
391# define KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE next_member_is_specialized = false;
392
393# define KERNEL_STRUCT_MEMBER(parent, _type, name) \
394 if (!next_member_is_specialized) { \
395 string_replace( \
396 *source, "kernel_data_" #parent "_" #name, "kernel_data." #parent ".__unused_" #name); \
397 next_member_is_specialized = true; \
398 }
399
400# include "kernel/data_template.h"
401
402# undef KERNEL_STRUCT_MEMBER
403# undef KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE
404# undef KERNEL_STRUCT_BEGIN
405
406 /* Replace "kernel_data.kernel_features" memory fetches with a function constant. */
408 *source, "kernel_data.kernel_features", "kernel_data_kernel_features");
409
410 metal_printf("KernelData patching took %.1f ms", (time_dt() - starttime) * 1000.0);
411 }
412
413 /* Opt in to all of available specializations. This can be made more granular for the
414 * PSO_SPECIALIZED_INTERSECT case in order to minimize the number of specialization requests,
415 * but the overhead should be negligible as these are very quick to (re)build and aren't
416 * serialized to disk via MTLBinaryArchives.
417 */
418 global_defines += "#define __KERNEL_USE_DATA_CONSTANTS__\n";
419 }
420
421 if (source) {
422 *source = global_defines + *source;
423 }
424
425 MD5Hash md5;
426 md5.append(global_defines);
427 return md5.get_hex();
428}
429
430void MetalDevice::make_source(MetalPipelineType pso_type, const uint64_t kernel_features)
431{
432 string &source = this->source[pso_type];
433 source = "\n#include \"kernel/device/metal/kernel.metal\"\n";
434 source = path_source_replace_includes(source, path_get("source"));
435
436 /* Perform any required specialization on the source.
437 * With Metal function constants we can generate a single variant of the kernel source which can
438 * be repeatedly respecialized.
439 */
440 global_defines_md5[pso_type] = preprocess_source(pso_type, kernel_features, &source);
441}
442
443bool MetalDevice::load_kernels(const uint64_t _kernel_features)
444{
445 @autoreleasepool {
446 kernel_features |= _kernel_features;
447
448 /* check if GPU is supported */
449 if (!support_device(kernel_features)) {
450 return false;
451 }
452
453 /* Keep track of whether motion blur is enabled, so to enable/disable motion in BVH builds
454 * This is necessary since objects may be reported to have motion if the Vector pass is
455 * active, but may still need to be rendered without motion blur if that isn't active as well.
456 */
457 motion_blur = motion_blur || (kernel_features & KERNEL_FEATURE_OBJECT_MOTION);
458
459 /* Only request generic kernels if they aren't cached in memory. */
460 refresh_source_and_kernels_md5(PSO_GENERIC);
461 if (MetalDeviceKernels::should_load_kernels(this, PSO_GENERIC)) {
462 /* If needed, load them asynchronously in order to responsively message progress to the user.
463 */
464 int this_device_id = this->device_id;
465 auto compile_kernels_fn = ^() {
466 compile_and_load(this_device_id, PSO_GENERIC);
467 };
468
469 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0),
470 compile_kernels_fn);
471 }
472 }
473 return true;
474}
475
476void MetalDevice::refresh_source_and_kernels_md5(MetalPipelineType pso_type)
477{
478 string defines_md5 = preprocess_source(pso_type, kernel_features);
479
480 /* Rebuild the source string if the injected block of #defines has changed. */
481 if (global_defines_md5[pso_type] != defines_md5) {
482 make_source(pso_type, kernel_features);
483 }
484
485 string constant_values;
486 if (pso_type != PSO_GENERIC) {
487 bool next_member_is_specialized = true;
488
489# define KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE next_member_is_specialized = false;
490
491 /* Add specialization constants to md5 so that 'get_best_pipeline' is able to return a suitable
492 * match. */
493# define KERNEL_STRUCT_MEMBER(parent, _type, name) \
494 if (next_member_is_specialized) { \
495 constant_values += string(#parent "." #name "=") + \
496 to_string(_type(launch_params->data.parent.name)) + "\n"; \
497 } \
498 else { \
499 next_member_is_specialized = true; \
500 }
501
502# include "kernel/data_template.h"
503
504# undef KERNEL_STRUCT_MEMBER
505# undef KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE
506 }
507
508 MD5Hash md5;
509 md5.append(constant_values);
510 md5.append(source[pso_type]);
511 if (use_metalrt) {
512 md5.append(string_printf("metalrt_features=%llu", kernel_features & METALRT_FEATURE_MASK));
513 }
514 if (pso_type != PSO_GENERIC) {
515 /* Include kernel_features since it's specialized but missed by the constant_values loop. */
516 md5.append(string_printf("kernel_features=%llu", launch_params->data.kernel_features));
517 }
518 kernels_md5[pso_type] = md5.get_hex();
519}
520
521void MetalDevice::compile_and_load(const int device_id, MetalPipelineType pso_type)
522{
523 @autoreleasepool {
524 /* Thread-safe front-end compilation. Typically the MSL->AIR compilation can take a few
525 * seconds, so we avoid blocking device tear-down if the user cancels a render immediately. */
526
527 id<MTLDevice> mtlDevice;
528 string source;
529
530 /* Safely gather any state required for the MSL->AIR compilation. */
531 {
532 thread_scoped_lock lock(existing_devices_mutex);
533
534 /* Check whether the device still exists. */
535 MetalDevice *instance = get_device_by_ID(device_id, lock);
536 if (!instance) {
537 metal_printf("Ignoring %s compilation request - device no longer exists",
538 kernel_type_as_string(pso_type));
539 return;
540 }
541
542 if (!MetalDeviceKernels::should_load_kernels(instance, pso_type)) {
543 /* We already have a full set of matching pipelines which are cached or queued. Return
544 * early to avoid redundant MTLLibrary compilation. */
545 metal_printf("Ignoreing %s compilation request - kernels already requested",
546 kernel_type_as_string(pso_type));
547 return;
548 }
549
550 mtlDevice = instance->mtlDevice;
551 source = instance->source[pso_type];
552 }
553
554 /* Perform the actual compilation using our cached context. The MetalDevice can safely destruct
555 * in this time. */
556
557 MTLCompileOptions *options = [[MTLCompileOptions alloc] init];
558
559 options.fastMathEnabled = YES;
560 options.languageVersion = MTLLanguageVersion3_0;
561# if defined(MAC_OS_VERSION_14_0)
562 if (@available(macos 14.0, *)) {
563 options.languageVersion = MTLLanguageVersion3_1;
564 }
565# endif
566# if defined(MAC_OS_VERSION_15_0)
567 if (@available(macos 15.0, *)) {
568 options.languageVersion = MTLLanguageVersion3_2;
569 if (const char *loglevel = getenv("MTL_LOG_LEVEL")) {
570 if (strcmp(loglevel, "MTLLogLevelDebug") == 0) {
571 options.enableLogging = true;
572 }
573 }
574 }
575# endif
576
577 if (getenv("CYCLES_METAL_PROFILING") || getenv("CYCLES_METAL_DEBUG")) {
578 path_write_text(path_cache_get(string_printf("%s.metal", kernel_type_as_string(pso_type))),
579 source);
580 }
581
582 double starttime = time_dt();
583
584 NSError *error = nullptr;
585 id<MTLLibrary> mtlLibrary = [mtlDevice newLibraryWithSource:@(source.c_str())
587 error:&error];
588
589 metal_printf("Front-end compilation finished in %.1f seconds (%s)",
590 time_dt() - starttime,
591 kernel_type_as_string(pso_type));
592
593 [options release];
594
595 bool blocking_pso_build = (getenv("CYCLES_METAL_PROFILING") ||
596 MetalDeviceKernels::is_benchmark_warmup());
597 if (blocking_pso_build) {
598 MetalDeviceKernels::wait_for_all();
599 starttime = 0.0;
600 }
601
602 /* Save the compiled MTLLibrary and trigger the AIR->PSO builds (if the MetalDevice still
603 * exists). */
604 {
605 thread_scoped_lock lock(existing_devices_mutex);
606 if (MetalDevice *instance = get_device_by_ID(device_id, lock)) {
607 if (mtlLibrary) {
608 if (error && [error localizedDescription]) {
609 LOG_WARNING << "MSL compilation messages: "
610 << [[error localizedDescription] UTF8String];
611 }
612
613 instance->mtlLibrary[pso_type] = mtlLibrary;
614
615 starttime = time_dt();
616 MetalDeviceKernels::load(instance, pso_type);
617 }
618 else {
619 NSString *err = [error localizedDescription];
620 instance->set_error(string_printf("Failed to compile library:\n%s", [err UTF8String]));
621 }
622 }
623 }
624
625 if (starttime && blocking_pso_build) {
626 MetalDeviceKernels::wait_for_all();
627
628 metal_printf("Back-end compilation finished in %.1f seconds (%s)",
629 time_dt() - starttime,
630 kernel_type_as_string(pso_type));
631 }
632 }
633}
634
635bool MetalDevice::is_texture(const KernelImageInfo &info)
636{
637 return info.height > 0;
638}
639
640void MetalDevice::erase_allocation(device_memory &mem)
641{
642 mem.device_pointer = 0;
643 mem.device_size = 0;
644
645 auto it = metal_mem_map.find(&mem);
646 if (it != metal_mem_map.end()) {
647 MetalMem *mmem = it->second.get();
648
649 /* blank out reference to resource in the launch params (fixes crash #94736) */
650 if (mmem->pointer_index >= 0) {
651 device_ptr *pointers = (device_ptr *)launch_params;
652 pointers[mmem->pointer_index] = 0;
653 }
654 metal_mem_map.erase(it);
655 }
656}
657
658bool MetalDevice::max_working_set_exceeded(const size_t safety_margin) const
659{
660 /* We're allowed to allocate beyond the safe working set size, but then if all resources are made
661 * resident we will get command buffer failures at render time. */
662 size_t available = [mtlDevice recommendedMaxWorkingSetSize] - safety_margin;
663 return (stats.mem_used > available);
664}
665
666MetalDevice::MetalMem *MetalDevice::generic_alloc(device_memory &mem)
667{
668 @autoreleasepool {
669 size_t size = mem.memory_size();
670
671 mem.device_pointer = 0;
672
673 id<MTLBuffer> metal_buffer = nil;
674 MTLResourceOptions options = MTLResourceStorageModeShared;
675
676 if (size > 0) {
677 if (mem.type == MEM_DEVICE_ONLY && !capture_enabled) {
678 options = MTLResourceStorageModePrivate;
679 }
680
681 metal_buffer = [mtlDevice newBufferWithLength:size options:options];
682
683 if (!metal_buffer) {
684 set_error("System is out of GPU memory");
685 return nullptr;
686 }
687 }
688
689 LOG_DEBUG << "Buffer allocate: " << mem.log_name() << ", "
690 << string_human_readable_number(mem.memory_size()) << " bytes. ("
692
693 mem.device_size = metal_buffer.allocatedSize;
694 metal_mem_alloc(metal_buffer);
695
696 metal_buffer.label = [NSString stringWithFormat:@"%s", mem.log_name().c_str()];
697
698 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
699
700 assert(metal_mem_map.count(&mem) == 0); /* assert against double-alloc */
701 unique_ptr<MetalMem> mmem = make_unique<MetalMem>();
702
703 mmem->mem = &mem;
704 mmem->mtlBuffer = metal_buffer;
705 mmem->offset = 0;
706 mmem->size = size;
707 if (options != MTLResourceStorageModePrivate) {
708 mmem->hostPtr = [metal_buffer contents];
709 }
710 else {
711 mmem->hostPtr = nullptr;
712 }
713
714 /* encode device_pointer as (MetalMem*) in order to handle resource relocation and device
715 * pointer recalculation */
716 mem.device_pointer = device_ptr(mmem.get());
717
718 if (metal_buffer.storageMode == MTLStorageModeShared) {
719 /* Replace host pointer with our host allocation. */
720 if (mem.host_pointer && mem.host_pointer != mmem->hostPtr) {
721 memcpy(mmem->hostPtr, mem.host_pointer, size);
722
723 host_free(mem.type, mem.host_pointer, mem.memory_size());
724 mem.host_pointer = mmem->hostPtr;
725 }
726 mem.shared_pointer = mmem->hostPtr;
727 mem.shared_counter++;
728 }
729
730 MetalMem *mmem_ptr = mmem.get();
731 metal_mem_map[&mem] = std::move(mmem);
732
733 if (max_working_set_exceeded()) {
734 set_error("System is out of GPU memory");
735 return nullptr;
736 }
737
738 return mmem_ptr;
739 }
740}
741
742void MetalDevice::generic_copy_to(device_memory & /*mem*/)
743{
744 /* No need to copy - Apple Silicon has Unified Memory Architecture. */
745}
746
747void MetalDevice::generic_free(device_memory &mem)
748{
749 if (!mem.device_pointer) {
750 return;
751 }
752
753 /* Host pointer should already have been freed at this point. If not we might
754 * end up freeing shared memory and can't recover original host memory.
755 * A separate host pointer remains after host_only_resize(). */
756 assert(mem.host_pointer == nullptr || mem.host_pointer != mem.shared_pointer);
757
758 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
759 MetalMem &mmem = *metal_mem_map.at(&mem);
760 size_t size = mmem.size;
761
762 bool free_mtlBuffer = true;
763
764 /* If this is shared, reference counting is used to safely free memory. */
765 if (mem.shared_pointer) {
766 assert(mem.shared_counter > 0);
767 if (--mem.shared_counter > 0) {
768 free_mtlBuffer = false;
769 }
770 }
771
772 if (free_mtlBuffer) {
773 if (mem.host_pointer && mem.host_pointer == mem.shared_pointer) {
774 /* Safely move the device-side data back to the host before it is freed.
775 * We should actually never reach this code as it is inefficient, but
776 * better than to crash if there is a bug. */
777 assert(!"Metal device should not copy memory back to host");
778 mem.host_pointer = mem.host_alloc(size);
779 memcpy(mem.host_pointer, mem.shared_pointer, size);
780 }
781
782 mem.shared_pointer = nullptr;
783
784 /* Free device memory. */
785 metal_mem_free(mmem.mtlBuffer);
786 mmem.mtlBuffer = nil;
787 }
788
789 erase_allocation(mem);
790}
791
792void MetalDevice::mem_alloc(device_memory &mem)
793{
794 if (mem.type == MEM_IMAGE_TEXTURE) {
795 assert(!"mem_alloc not supported for images.");
796 }
797 else if (mem.type == MEM_GLOBAL) {
798 generic_alloc(mem);
799 }
800 else {
801 generic_alloc(mem);
802 }
803}
804
805void MetalDevice::mem_copy_to(device_memory &mem)
806{
807 if (!mem.device_pointer) {
808 if (mem.type == MEM_GLOBAL) {
809 global_alloc(mem);
810 }
811 else if (mem.type == MEM_IMAGE_TEXTURE) {
812 image_alloc((device_image &)mem);
813 }
814 else {
815 generic_alloc(mem);
816 generic_copy_to(mem);
817 }
818 }
819 else if (mem.is_resident(this)) {
820 if (mem.type == MEM_GLOBAL) {
821 generic_copy_to(mem);
822 }
823 else if (mem.type == MEM_IMAGE_TEXTURE) {
824 image_copy_to((device_image &)mem);
825 }
826 else {
827 generic_copy_to(mem);
828 }
829 }
830}
831
832void MetalDevice::mem_move_to_host(device_memory & /*mem*/)
833{
834 /* Metal implements own mechanism for moving host memory. */
835 assert(!"Metal does not support mem_move_to_host");
836}
837
838void MetalDevice::mem_copy_from(
839 device_memory & /*mem*/, const size_t /*y*/, size_t /*w*/, const size_t /*h*/, size_t /*elem*/)
840{
841 /* No need to copy - Apple Silicon has Unified Memory Architecture. */
842}
843
844void MetalDevice::mem_or_from_device(device_memory & /*mem*/)
845{
846 /* No need to copy - Apple Silicon has Unified Memory Architecture. */
847}
848
849void MetalDevice::mem_zero(device_memory &mem)
850{
851 if (!mem.device_pointer) {
852 mem_alloc(mem);
853 }
855 memset(mem.shared_pointer, 0, mem.memory_size());
856}
857
858void MetalDevice::mem_free(device_memory &mem)
859{
860 if (mem.type == MEM_GLOBAL) {
861 global_free(mem);
862 }
863 else if (mem.type == MEM_IMAGE_TEXTURE) {
864 image_free((device_image &)mem);
865 }
866 else {
867 generic_free(mem);
868 }
869}
870
871device_ptr MetalDevice::mem_alloc_sub_ptr(device_memory & /*mem*/,
872 size_t /*offset*/,
873 size_t /*size*/)
874{
875 /* METAL_WIP - revive if necessary */
876 assert(0);
877 return 0;
878}
879
880void MetalDevice::cancel()
881{
882 /* Remove this device's ID from the list of active devices. Any pending compilation requests
883 * originating from this session will be cancelled. */
884 thread_scoped_lock lock(existing_devices_mutex);
885 if (device_id) {
886 active_device_ids.erase(device_id);
887 device_id = 0;
888 }
889}
890
891bool MetalDevice::is_ready(string &status) const
892{
893 if (!error_msg.empty()) {
894 /* Avoid hanging if we had an error. */
895 return true;
896 }
897
898 int num_loaded = MetalDeviceKernels::get_loaded_kernel_count(this, PSO_GENERIC);
899 if (num_loaded < DEVICE_KERNEL_NUM) {
900 status = string_printf("%d / %d render kernels loaded (may take a few minutes the first time)",
901 num_loaded,
903 return false;
904 }
905
906 if (int num_requests = MetalDeviceKernels::num_incomplete_specialization_requests()) {
907 status = string_printf("%d kernels to optimize", num_requests);
908 }
909 else if (kernel_specialization_level == PSO_SPECIALIZED_INTERSECT) {
910 status = "Using optimized intersection kernels";
911 }
912 else if (kernel_specialization_level == PSO_SPECIALIZED_SHADE) {
913 status = "Using optimized kernels";
914 }
915
916 metal_printf("MetalDevice::is_ready(...) --> true");
917 return true;
918}
919
920bool MetalDevice::set_bvh_limits(size_t instance_count, size_t max_prim_count)
921{
922 /* For object & primitive counts above a certain limit, MetalRT requires extended limits to be
923 * built into the kernels, and when building BVHs. Following best practices, this should only
924 * be enabled when necessary. See
925 * https://developer.apple.com/documentation/metal/mtlaccelerationstructureusage/mtlaccelerationstructureusageextendedlimits?language=objc
926 */
927
928 const int standard_limits_max_prim_count = (1 << 28);
929 const int standard_limits_max_instance_count = (1 << 24);
930
931 bool using_metalrt_extended_limits_before = use_metalrt_extended_limits;
932
933 /* Enable extended limits if object count exceeds max supported by standard limits.
934 * Once enabled, it remains enabled for the lifetime of the device. */
935 if (instance_count > standard_limits_max_instance_count ||
936 max_prim_count > standard_limits_max_prim_count)
937 {
938 use_metalrt_extended_limits = true;
939 metal_printf("Enabling MetalRT extended limits (max_prim_count = %zu, instance_count = %zu)",
940 max_prim_count,
941 instance_count);
942 }
943
944 /* All BVHs need to be rebuilt if the extended limits state changes. */
945 return using_metalrt_extended_limits_before != use_metalrt_extended_limits;
946}
947
948void MetalDevice::optimize_for_scene(Scene *scene)
949{
950 MetalPipelineType specialization_level = kernel_specialization_level;
951
952 if (!scene->params.background) {
953 /* In live viewport, don't specialize beyond intersection kernels for responsiveness. */
954 specialization_level = (MetalPipelineType)min(specialization_level, PSO_SPECIALIZED_INTERSECT);
955 }
956
957 /* For responsive rendering, specialize the kernels in the background, and only if there isn't an
958 * existing "optimize_for_scene" request in flight. */
959 int this_device_id = this->device_id;
960 auto specialize_kernels_fn = ^() {
961 for (int level = 1; level <= int(specialization_level); level++) {
962 compile_and_load(this_device_id, MetalPipelineType(level));
963 }
964 };
965
966 /* In normal use, we always compile the specialized kernels in the background. */
967 bool specialize_in_background = true;
968
969 /* Block if a per-kernel profiling is enabled (ensure steady rendering rate). */
970 if (getenv("CYCLES_METAL_PROFILING") != nullptr) {
971 specialize_in_background = false;
972 }
973
974 /* Block during benchmark warm-up to ensure kernels are cached prior to the observed run. */
975 if (MetalDeviceKernels::is_benchmark_warmup()) {
976 specialize_in_background = false;
977 }
978
979 if (specialize_in_background) {
980 if (MetalDeviceKernels::num_incomplete_specialization_requests() == 0) {
981 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0),
982 specialize_kernels_fn);
983 }
984 else {
985 metal_printf("\"optimize_for_scene\" request already in flight - dropping request");
986 }
987 }
988 else {
989 specialize_kernels_fn();
990 }
991}
992
993void MetalDevice::const_copy_to(const char *name, void *host, const size_t size)
994{
995 if (strcmp(name, "data") == 0) {
996 assert(size == sizeof(KernelData));
997 memcpy((uint8_t *)&launch_params->data, host, sizeof(KernelData));
998
999 /* Refresh the kernels_md5 checksums for specialized kernel sets. */
1000 for (int level = 1; level <= int(kernel_specialization_level); level++) {
1001 refresh_source_and_kernels_md5(MetalPipelineType(level));
1002 }
1003 return;
1004 }
1005
1006 auto update_launch_pointers = [&](size_t offset, void *data, const size_t pointers_size) {
1007 uint64_t *addresses = (uint64_t *)((uint8_t *)launch_params + offset);
1008
1009 MetalMem **mmem = (MetalMem **)data;
1010 int pointer_count = pointers_size / sizeof(device_ptr);
1011 int pointer_index = offset / sizeof(device_ptr);
1012 for (int i = 0; i < pointer_count; i++) {
1013 addresses[i] = 0;
1014 if (mmem[i]) {
1015 mmem[i]->pointer_index = pointer_index + i;
1016 if (mmem[i]->mtlBuffer) {
1017 addresses[i] = mmem[i]->mtlBuffer.gpuAddress;
1018 }
1019 }
1020 }
1021 };
1022
1023 /* Update data storage pointers in launch parameters. */
1024 if (strcmp(name, "integrator_state") == 0) {
1025 /* IntegratorStateGPU is contiguous pointers up until sort_partition_divisor. */
1026 const size_t pointer_block_size = offsetof(IntegratorStateGPU, sort_partition_divisor);
1027 update_launch_pointers(
1028 offsetof(KernelParamsMetal, integrator_state), host, pointer_block_size);
1029
1030 /* Ensure the non-pointers part of IntegratorStateGPU is copied (this is the proper fix for
1031 * #144713). */
1032 memcpy((uint8_t *)&launch_params->integrator_state + pointer_block_size,
1033 (uint8_t *)host + pointer_block_size,
1034 sizeof(IntegratorStateGPU) - pointer_block_size);
1035 }
1036# define KERNEL_DATA_ARRAY(data_type, tex_name) \
1037 else if (strcmp(name, #tex_name) == 0) \
1038 { \
1039 update_launch_pointers(offsetof(KernelParamsMetal, tex_name), host, size); \
1040 }
1041# include "kernel/data_arrays.h"
1042# undef KERNEL_DATA_ARRAY
1043}
1044
1045void MetalDevice::global_alloc(device_memory &mem)
1046{
1047 if (mem.is_resident(this)) {
1048 generic_alloc(mem);
1049 generic_copy_to(mem);
1050 }
1051
1052 const_copy_to(mem.global_name(), &mem.device_pointer, sizeof(mem.device_pointer));
1053}
1054
1055void MetalDevice::global_free(device_memory &mem)
1056{
1057 if (mem.is_resident(this) && mem.device_pointer) {
1058 generic_free(mem);
1059 }
1060}
1061
1062void MetalDevice::image_alloc_as_buffer(device_image &mem)
1063{
1064 MetalDevice::MetalMem *mmem = generic_alloc(mem);
1065 generic_copy_to(mem);
1066
1067 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
1068
1069 /* Resize once. Only on the host, kernels may be executing. The device buffer is reallocated
1070 * in MetalDeviceQueue::load_image_info(). */
1071 const uint image_info_id = mem.image_info_id;
1072 if (image_info_id >= image_info.size()) {
1073 /* Geometric growth to amortize reallocation cost. */
1074 const size_t new_size = max(size_t(image_info_id) + 128, image_info.size() * 2);
1075 image_info.host_only_resize(new_size);
1076 image_info_id_map.resize(new_size);
1077 }
1078
1079 image_info[image_info_id] = mem.info;
1080 image_info_id_map[image_info_id] = mmem->mtlBuffer;
1081
1082 if (is_nanovdb_type(mem.info.data_type)) {
1083 using_nanovdb = true;
1084 }
1085}
1086
1087void MetalDevice::image_alloc(device_image &mem)
1088{
1089 @autoreleasepool {
1090 /* Check that dimensions fit within maximum allowable size.
1091 * If 1D texture is allocated, use 1D buffer.
1092 * See: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf */
1093 if (mem.data_height > 0) {
1094 if (mem.data_width > 16384 || mem.data_height > 16384) {
1095 set_error(string_printf(
1096 "Texture exceeds maximum allowed size of 16384 x 16384 (requested: %zu x %zu)",
1097 mem.data_width,
1098 mem.data_height));
1099 return;
1100 }
1101 }
1102
1103 /* General variables for both architectures */
1104 size_t size = mem.memory_size();
1105
1106 /* sampler_index maps into the GPU's constant 'metal_samplers' array */
1107 uint64_t sampler_index = mem.info.extension;
1109 sampler_index += 4;
1110 }
1111
1112 /* Image Texture Storage */
1113 MTLPixelFormat format;
1114 switch (mem.data_type) {
1115 case TYPE_UCHAR: {
1116 MTLPixelFormat formats[] = {MTLPixelFormatR8Unorm,
1117 MTLPixelFormatRG8Unorm,
1118 MTLPixelFormatInvalid,
1119 MTLPixelFormatRGBA8Unorm};
1120 format = formats[mem.data_elements - 1];
1121 } break;
1122 case TYPE_UINT16: {
1123 MTLPixelFormat formats[] = {MTLPixelFormatR16Unorm,
1124 MTLPixelFormatRG16Unorm,
1125 MTLPixelFormatInvalid,
1126 MTLPixelFormatRGBA16Unorm};
1127 format = formats[mem.data_elements - 1];
1128 } break;
1129 case TYPE_UINT: {
1130 MTLPixelFormat formats[] = {MTLPixelFormatR32Uint,
1131 MTLPixelFormatRG32Uint,
1132 MTLPixelFormatInvalid,
1133 MTLPixelFormatRGBA32Uint};
1134 format = formats[mem.data_elements - 1];
1135 } break;
1136 case TYPE_INT: {
1137 MTLPixelFormat formats[] = {MTLPixelFormatR32Sint,
1138 MTLPixelFormatRG32Sint,
1139 MTLPixelFormatInvalid,
1140 MTLPixelFormatRGBA32Sint};
1141 format = formats[mem.data_elements - 1];
1142 } break;
1143 case TYPE_FLOAT: {
1144 MTLPixelFormat formats[] = {MTLPixelFormatR32Float,
1145 MTLPixelFormatRG32Float,
1146 MTLPixelFormatInvalid,
1147 MTLPixelFormatRGBA32Float};
1148 format = formats[mem.data_elements - 1];
1149 } break;
1150 case TYPE_HALF: {
1151 MTLPixelFormat formats[] = {MTLPixelFormatR16Float,
1152 MTLPixelFormatRG16Float,
1153 MTLPixelFormatInvalid,
1154 MTLPixelFormatRGBA16Float};
1155 format = formats[mem.data_elements - 1];
1156 } break;
1157 default:
1158 assert(0);
1159 return;
1160 }
1161
1162 assert(format != MTLPixelFormatInvalid);
1163
1164 id<MTLTexture> mtlTexture = nil;
1165 size_t src_pitch = mem.data_width * datatype_size(mem.data_type) * mem.data_elements;
1166
1167 if (mem.data_height > 0) {
1168 /* 2D texture */
1169 MTLTextureDescriptor *desc;
1170
1171 desc = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format
1172 width:mem.data_width
1173 height:mem.data_height
1174 mipmapped:NO];
1175
1176 desc.storageMode = MTLStorageModeShared;
1177 desc.usage = MTLTextureUsageShaderRead;
1178
1179 /* Disallow lossless texture compression. Path-tracing texture access patterns are very
1180 * random, and cache reuse gains are typically too low to offset the decompression overheads.
1181 */
1182 desc.allowGPUOptimizedContents = false;
1183
1184 LOG_DEBUG << "Texture 2D allocate: " << mem.log_name() << ", "
1185 << string_human_readable_number(mem.memory_size()) << " bytes. ("
1186 << string_human_readable_size(mem.memory_size()) << ")";
1187
1188 mtlTexture = [mtlDevice newTextureWithDescriptor:desc];
1189 if (!mtlTexture) {
1190 set_error("System is out of GPU memory");
1191 return;
1192 }
1193
1194 [mtlTexture replaceRegion:MTLRegionMake2D(0, 0, mem.data_width, mem.data_height)
1195 mipmapLevel:0
1196 withBytes:mem.host_pointer
1197 bytesPerRow:src_pitch];
1198 }
1199 else {
1200 /* 1D image, using linear memory. */
1201 image_alloc_as_buffer(mem);
1202 return;
1203 }
1204
1205 mem.device_pointer = (device_ptr)mtlTexture;
1206 mem.device_size = size;
1207 metal_mem_alloc(mtlTexture);
1208
1209 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
1210 unique_ptr<MetalMem> mmem = make_unique<MetalMem>();
1211 mmem->mem = &mem;
1212 mmem->mtlTexture = mtlTexture;
1213 metal_mem_map[&mem] = std::move(mmem);
1214
1215 /* Resize once. Only on the host, kernels may be executing. The device buffers are
1216 * reallocated in MetalDeviceQueue::load_image_info(). */
1217 const uint image_info_id = mem.image_info_id;
1218 if (image_info_id >= image_info.size()) {
1219 /* Geometric growth to amortize reallocation cost. */
1220 const size_t new_size = max(size_t(image_info_id) + 128, image_info.size() * 2);
1221 image_info.host_only_resize(new_size);
1222 image_info_id_map.resize(new_size);
1223 }
1224
1225 /* Set Mapping. */
1226 image_info_id_map[image_info_id] = mtlTexture;
1227 image_info[image_info_id] = mem.info;
1228 image_info[image_info_id].data = uint64_t(image_info_id) | (sampler_index << 32);
1229
1230 if (max_working_set_exceeded()) {
1231 set_error("System is out of GPU memory");
1232 }
1233 }
1234}
1235
1236void MetalDevice::image_copy_to(device_image &mem)
1237{
1238 if (mem.is_resident(this)) {
1239 const size_t src_pitch = mem.data_width * datatype_size(mem.data_type) * mem.data_elements;
1240
1241 if (mem.data_height > 0) {
1242 id<MTLTexture> mtlTexture;
1243 {
1244 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
1245 mtlTexture = metal_mem_map.at(&mem)->mtlTexture;
1246 }
1247 [mtlTexture replaceRegion:MTLRegionMake2D(0, 0, mem.data_width, mem.data_height)
1248 mipmapLevel:0
1249 withBytes:mem.host_pointer
1250 bytesPerRow:src_pitch];
1251 }
1252 else {
1253 generic_copy_to(mem);
1254 }
1255 }
1256}
1257
1258void MetalDevice::image_free(device_image &mem)
1259{
1260 int image_info_id = mem.image_info_id;
1261 if (mem.data_height == 0) {
1262 generic_free(mem);
1263 }
1264 else if (metal_mem_map.contains(&mem)) {
1265 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
1266 MetalMem &mmem = *metal_mem_map.at(&mem);
1267
1268 /* Free bindless texture. */
1269 metal_mem_free(mmem.mtlTexture);
1270 mmem.mtlTexture = nil;
1271 erase_allocation(mem);
1272 }
1273 image_info_id_map[image_info_id] = nil;
1274}
1275
1276unique_ptr<DeviceQueue> MetalDevice::gpu_queue_create()
1277{
1278 return make_unique<MetalDeviceQueue>(this);
1279}
1280
1281bool MetalDevice::should_use_graphics_interop(const GraphicsInteropDevice &interop_device,
1282 const bool /*log*/)
1283{
1284 /* Always supported with unified memory. */
1285 return interop_device.type == GraphicsInteropDevice::METAL;
1286}
1287
1288void *MetalDevice::get_native_buffer(device_ptr ptr)
1289{
1290 return ((MetalMem *)ptr)->mtlBuffer;
1291}
1292
1293void MetalDevice::flush_delayed_free_list()
1294{
1295 /* free any Metal buffers that may have been freed by host while a command
1296 * buffer was being generated. This function should be called after each
1297 * completion of a command buffer */
1298 std::lock_guard<std::recursive_mutex> lock(metal_mem_map_mutex);
1299 for (auto &it : delayed_free_list) {
1300 [it release];
1301 }
1302 delayed_free_list.clear();
1303}
1304
1305void MetalDevice::build_bvh(BVH *bvh, Progress &progress, bool refit)
1306{
1307 @autoreleasepool {
1308 if (bvh->params.bvh_layout == BVH_LAYOUT_BVH2) {
1309 Device::build_bvh(bvh, progress, refit);
1310 return;
1311 }
1312
1313 BVHMetal *bvh_metal = static_cast<BVHMetal *>(bvh);
1314 bvh_metal->motion_blur = motion_blur;
1315 bvh_metal->use_pcmi = use_pcmi;
1316 bvh_metal->extended_limits = use_metalrt_extended_limits;
1317 if (bvh_metal->build(progress, mtlDevice, mtlGeneralCommandQueue, refit)) {
1318
1319 if (bvh->params.top_level) {
1320 update_bvh(bvh_metal);
1321 }
1322 }
1323
1324 if (max_working_set_exceeded()) {
1325 set_error("System is out of GPU memory");
1326 }
1327 }
1328}
1329
1330void MetalDevice::free_bvh()
1331{
1332 /* metal_mem_free defers the actual release via delayed_free_list,
1333 * since the old BVH may still be referenced by an in-flight command buffer. */
1334 for (id<MTLAccelerationStructure> &blas : unique_blas_array) {
1335 metal_mem_free(blas);
1336 }
1337 unique_blas_array.clear();
1338 blas_array.clear();
1339
1340 if (blas_buffer) {
1341 metal_mem_free(blas_buffer);
1342 blas_buffer = nil;
1343 }
1344
1345 if (accel_struct) {
1346 metal_mem_free(accel_struct);
1347 accel_struct = nil;
1348 }
1349}
1350
1351void MetalDevice::update_bvh(BVHMetal *bvh_metal)
1352{
1353 free_bvh();
1354
1355 if (!bvh_metal) {
1356 return;
1357 }
1358
1359 accel_struct = bvh_metal->accel_struct;
1360 unique_blas_array = bvh_metal->unique_blas_array;
1361 blas_array = bvh_metal->blas_array;
1362
1363 /* Memory tracking and residency are managed here (not in BVHMetal::set_accel_struct)
1364 * to pair with free_bvh and reflect actual device ownership. */
1365 metal_mem_alloc(accel_struct);
1366
1367 [accel_struct retain];
1368 for (id<MTLAccelerationStructure> &blas : unique_blas_array) {
1369 [blas retain];
1370 metal_mem_alloc(blas);
1371 }
1372
1373 // Allocate required buffers for BLAS array.
1374 uint64_t buffer_size = blas_array.size() * sizeof(uint64_t);
1375 blas_buffer = [mtlDevice newBufferWithLength:buffer_size options:MTLResourceStorageModeShared];
1376 metal_mem_alloc(blas_buffer);
1377}
1378
1380
1381#endif
volatile int lock
void init()
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition btDbvt.cpp:52
btScalar max
void refit(btStridingMeshInterface *triangles, const btVector3 &aabbMin, const btVector3 &aabbMax)
BVHLayout bvh_layout
Definition params.h:83
bool top_level
Definition params.h:80
Definition bvh/bvh.h:67
BVHParams params
Definition bvh/bvh.h:69
Metal metal
Definition debug.h:155
KernelOptimizationLevel kernel_optimization_level
string description
bool use_hardware_raytracing
virtual void build_bvh(BVH *bvh, Progress &progress, bool refit)
virtual void set_error(const string &error)
Definition md5.h:19
void append(const uint8_t *data, const int nbytes)
Definition md5.cpp:268
string get_hex()
Definition md5.cpp:367
bool background
Definition scene.h:91
Definition scene.h:133
SceneParams params
Definition scene.h:190
size_t mem_used
Definition util/stats.h:30
void mem_alloc(const size_t size)
Definition util/stats.h:18
void mem_free(const size_t size)
Definition util/stats.h:24
KernelImageInfo info
string log_name() const override
Definition memory.cpp:280
bool is_resident(Device *sub_device) const
Definition memory.cpp:193
void * host_alloc(const size_t size)
Definition memory.cpp:93
const char * global_name() const
Definition memory.cpp:83
virtual string log_name() const
Definition memory.cpp:88
@ MEM_IMAGE_TEXTURE
@ MEM_DEVICE_ONLY
@ TYPE_UINT16
CCL_NAMESPACE_BEGIN struct Options options
#define KERNEL_FEATURE_OBJECT_MOTION
#define CCL_NAMESPACE_END
@ KERNEL_OPTIMIZATION_LEVEL_OFF
@ KERNEL_OPTIMIZATION_LEVEL_FULL
@ KERNEL_OPTIMIZATION_LEVEL_INTERSECT
#define offsetof(t, d)
#define str(s)
#define assert(assertion)
@ BVH_LAYOUT_METAL
@ BVH_LAYOUT_BVH2
@ DEVICE_KERNEL_NUM
#define LOG_DEBUG
Definition log.h:107
#define LOG_ERROR
Definition log.h:101
#define LOG_WARNING
Definition log.h:103
unsigned int uint
Definition BKE_text.h:55
BMesh const char void * data
std::string format(const std::string &fmt, Args &&...args)
Definition diagnostic.hh:43
int BVHLayoutMask
Definition params.h:50
string path_cache_get(const string &sub)
Definition path.cpp:383
string path_source_replace_includes(const string &source, const string &path)
Definition path.cpp:1017
string path_get(const string &sub)
Definition path.cpp:368
bool path_write_text(const string &path, const string &text)
Definition path.cpp:722
#define min(a, b)
Definition sort.cc:38
string string_human_readable_size(size_t size)
Definition string.cpp:333
string to_string(const char *str)
Definition string.cpp:202
string string_human_readable_number(size_t num)
Definition string.cpp:352
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
Definition string.cpp:25
void string_replace_same_length(string &haystack, const string &needle, const string &other)
Definition string.cpp:157
bool use_metalrt_pcmi
Definition debug.h:108
bool adaptive_compile
Definition debug.h:96
bool use_local_atomic_sort
Definition debug.h:99
std::mutex thread_mutex
Definition thread.h:27
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
CCL_NAMESPACE_BEGIN double time_dt()
Definition time.cpp:31
uint64_t device_ptr
Definition types_base.h:44
@ INTERPOLATION_CLOSEST
Definition types_image.h:27
ccl_device_inline bool is_nanovdb_type(int type)
Definition types_image.h:54