22# include <TargetConditionals.h>
23# include <crt_externs.h>
30std::map<int, MetalDevice *> MetalDevice::active_device_ids;
34MetalDevice *MetalDevice::get_device_by_ID(
const int ID,
37 auto it = active_device_ids.find(ID);
38 if (it != active_device_ids.end()) {
44bool MetalDevice::is_device_cancelled(
const int ID)
47 return get_device_by_ID(ID,
lock) ==
nullptr;
55void MetalDevice::set_error(
const string &error)
57 static std::mutex s_error_mutex;
58 std::lock_guard<std::mutex>
lock(s_error_mutex);
63 LOG_ERROR <<
"Refer to the Cycles GPU rendering documentation for possible solutions:\n"
64 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
70 :
Device(info, stats, profiler, headless), image_info(this,
"image_info",
MEM_GLOBAL)
77 static int existing_devices_counter = 1;
78 device_id = existing_devices_counter++;
79 active_device_ids[device_id] =
this;
85 auto usable_devices = MetalInfo::get_usable_devices();
86 assert(mtlDevId < usable_devices.size());
87 mtlDevice = usable_devices[mtlDevId];
88 metal_printf(
"Creating new Cycles Metal device: %s", info.
description.c_str());
92 if (@available(macOS 13.3, *)) {
93 [mtlDevice setShouldMaximizeConcurrentCompilation:YES];
96 max_threads_per_threadgroup = 512;
99 if (
const char *metalrt = getenv(
"CYCLES_METALRT")) {
100 use_metalrt = (atoi(metalrt) != 0);
103 if (
const char *
str = getenv(
"CYCLES_METALRT_EXTENDED_LIMITS")) {
104 use_metalrt_extended_limits = (atoi(
str) != 0);
107# if defined(MAC_OS_VERSION_15_0)
110 if (use_metalrt && [mtlDevice supportsFamily:MTLGPUFamilyApple9]) {
112 if (@available(macos 15.6, *)) {
118 if (getenv(
"CYCLES_DEBUG_METAL_CAPTURE_KERNEL")) {
119 capture_enabled =
true;
125 if (
auto *
str = getenv(
"CYCLES_METAL_PROFILING")) {
126 if (atoi(
str) && [mtlDevice supportsCounterSampling:MTLCounterSamplingPointAtStageBoundary])
128 NSArray<id<MTLCounterSet>> *counterSets = [mtlDevice counterSets];
130 NSError *error = nil;
131 MTLCounterSampleBufferDescriptor *desc = [[MTLCounterSampleBufferDescriptor alloc]
init];
132 [desc setStorageMode:MTLStorageModeShared];
133 [desc setLabel:
@"CounterSampleBuffer"];
134 [desc setSampleCount:MAX_SAMPLE_BUFFER_LENGTH];
135 [desc setCounterSet:counterSets[0]];
136 mtlCounterSampleBuffer = [mtlDevice newCounterSampleBufferWithDescriptor:desc
138 [mtlCounterSampleBuffer retain];
145 kernel_specialization_level = PSO_GENERIC;
149 kernel_specialization_level = PSO_SPECIALIZED_INTERSECT;
152 kernel_specialization_level = PSO_SPECIALIZED_SHADE;
156 if (
auto *envstr = getenv(
"CYCLES_METAL_SPECIALIZATION_LEVEL")) {
157 kernel_specialization_level = (MetalPipelineType)atoi(envstr);
159 metal_printf(
"kernel_specialization_level = %s",
160 kernel_type_as_string(
161 (MetalPipelineType)
min((
int)kernel_specialization_level, (
int)PSO_NUM - 1)));
167 mtlComputeCommandQueue = [mtlDevice newCommandQueue];
170 mtlGeneralCommandQueue = [mtlDevice newCommandQueue];
172# if defined(MAC_OS_VERSION_15_0)
173 if (@available(macos 15.0, *)) {
174 if (
DebugFlags().metal.use_residency_sets_if_available) {
177 MTLResidencySetDescriptor *residency_set_desc = [[MTLResidencySetDescriptor alloc]
init];
178 residency_set_desc.label =
@"CyclesResidencySet";
179 residency_set_desc.initialCapacity = 512;
180 NSError *error = nil;
181 mtlResidencySet = [mtlDevice newResidencySetWithDescriptor:residency_set_desc
183 [residency_set_desc release];
187 if (mtlResidencySet) {
188 mtlResidencySet_enabled =
true;
189 [mtlComputeCommandQueue addResidencySet:mtlResidencySet];
192 metal_printf(
"Failed to create residency set: %s",
193 [[error localizedDescription] UTF8String]);
199 image_bindings = [mtlDevice newBufferWithLength:8192
options:MTLResourceStorageModeShared];
200 metal_mem_alloc(image_bindings);
203 options:MTLResourceStorageModeShared];
204 metal_mem_alloc(launch_params_buffer);
211MetalDevice::~MetalDevice()
221 for (
int res = 0; res < image_info.size(); res++) {
222 metal_mem_free(image_info_id_map[res]);
223 image_info_id_map[res] = nil;
228 metal_mem_free(launch_params_buffer);
229 metal_mem_free(image_bindings);
231 flush_delayed_free_list();
233# if defined(MAC_OS_VERSION_15_0)
234 if (@available(macos 15.0, *)) {
235 if (mtlResidencySet) {
236 [mtlResidencySet endResidency];
237 [mtlResidencySet release];
242 [mtlComputeCommandQueue release];
243 [mtlGeneralCommandQueue release];
244 if (mtlCounterSampleBuffer) {
245 [mtlCounterSampleBuffer release];
250void MetalDevice::add_to_residency_set(id<MTLResource> allocation)
252# if defined(MAC_OS_VERSION_15_0)
253 if (@available(macos 15.0, *)) {
254 if (allocation && mtlResidencySet) {
255 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
256 [mtlResidencySet addAllocation:allocation];
257 mtlResidencySet_dirty =
true;
263void MetalDevice::remove_from_residency_set(id<MTLResource> allocation)
265# if defined(MAC_OS_VERSION_15_0)
266 if (@available(macos 15.0, *)) {
267 if (allocation && mtlResidencySet) {
268 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
269 [mtlResidencySet removeAllocation:allocation];
270 mtlResidencySet_dirty =
true;
276void MetalDevice::metal_mem_alloc(id<MTLResource> allocation)
279 stats.
mem_alloc(allocation.allocatedSize);
280 add_to_residency_set(allocation);
284void MetalDevice::metal_mem_free(id<MTLResource> allocation)
287 stats.
mem_free(allocation.allocatedSize);
289 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
293 remove_from_residency_set(allocation);
297 delayed_free_list.push_back(allocation);
301void MetalDevice::prepare_residency()
303# if defined(MAC_OS_VERSION_15_0)
304 if (@available(macos 15.0, *)) {
305 if (mtlResidencySet) {
306 std::lock_guard<std::mutex> residency_lock(mtlResidencySet_mutex);
307 if (mtlResidencySet_dirty) {
308 mtlResidencySet_dirty =
false;
309 [mtlResidencySet commit];
316bool MetalDevice::support_device(
const uint64_t )
321bool MetalDevice::check_peer_access(
Device * )
328bool MetalDevice::use_adaptive_compilation()
333bool MetalDevice::use_local_atomic_sort()
const
338string MetalDevice::preprocess_source(MetalPipelineType pso_type,
342 string global_defines;
343 if (use_adaptive_compilation()) {
344 global_defines +=
"#define __KERNEL_FEATURES__ " +
to_string(kernel_features) +
"\n";
348 global_defines +=
"#define __KERNEL_METALRT__\n";
350 global_defines +=
"#define __METALRT_MOTION__\n";
352 if (use_metalrt_extended_limits) {
353 global_defines +=
"#define __METALRT_EXTENDED_LIMITS__\n";
357# ifdef WITH_CYCLES_DEBUG
358 global_defines +=
"#define WITH_CYCLES_DEBUG\n";
361 global_defines +=
"#define __KERNEL_METAL_APPLE__\n";
362 if (@available(macos 14.0, *)) {
364 global_defines +=
"#define __METAL_GLOBAL_BUILTINS__\n";
369 if ((pso_type == PSO_GENERIC || using_nanovdb) &&
DebugFlags().metal.use_nanovdb) {
370 global_defines +=
"#define WITH_NANOVDB\n";
375 global_defines +=
"#define __KERNEL_METAL_TARGET_CPU_ARM64__\n";
382 if (pso_type != PSO_GENERIC) {
384 const double starttime =
time_dt();
386# define KERNEL_STRUCT_BEGIN(name, parent) \
387 string_replace_same_length(*source, "kernel_data." #parent ".", "kernel_data_" #parent "_");
389 bool next_member_is_specialized =
true;
391# define KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE next_member_is_specialized = false;
393# define KERNEL_STRUCT_MEMBER(parent, _type, name) \
394 if (!next_member_is_specialized) { \
396 *source, "kernel_data_" #parent "_" #name, "kernel_data." #parent ".__unused_" #name); \
397 next_member_is_specialized = true; \
402# undef KERNEL_STRUCT_MEMBER
403# undef KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE
404# undef KERNEL_STRUCT_BEGIN
408 *source,
"kernel_data.kernel_features",
"kernel_data_kernel_features");
410 metal_printf(
"KernelData patching took %.1f ms", (
time_dt() - starttime) * 1000.0);
418 global_defines +=
"#define __KERNEL_USE_DATA_CONSTANTS__\n";
422 *source = global_defines + *source;
426 md5.
append(global_defines);
430void MetalDevice::make_source(MetalPipelineType pso_type,
const uint64_t kernel_features)
432 string &source = this->source[pso_type];
433 source =
"\n#include \"kernel/device/metal/kernel.metal\"\n";
440 global_defines_md5[pso_type] = preprocess_source(pso_type, kernel_features, &source);
443bool MetalDevice::load_kernels(
const uint64_t _kernel_features)
446 kernel_features |= _kernel_features;
449 if (!support_device(kernel_features)) {
460 refresh_source_and_kernels_md5(PSO_GENERIC);
461 if (MetalDeviceKernels::should_load_kernels(
this, PSO_GENERIC)) {
464 int this_device_id = this->device_id;
465 auto compile_kernels_fn = ^() {
466 compile_and_load(this_device_id, PSO_GENERIC);
469 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0),
476void MetalDevice::refresh_source_and_kernels_md5(MetalPipelineType pso_type)
478 string defines_md5 = preprocess_source(pso_type, kernel_features);
481 if (global_defines_md5[pso_type] != defines_md5) {
482 make_source(pso_type, kernel_features);
485 string constant_values;
486 if (pso_type != PSO_GENERIC) {
487 bool next_member_is_specialized =
true;
489# define KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE next_member_is_specialized = false;
493# define KERNEL_STRUCT_MEMBER(parent, _type, name) \
494 if (next_member_is_specialized) { \
495 constant_values += string(#parent "." #name "=") + \
496 to_string(_type(launch_params->data.parent.name)) + "\n"; \
499 next_member_is_specialized = true; \
504# undef KERNEL_STRUCT_MEMBER
505# undef KERNEL_STRUCT_MEMBER_DONT_SPECIALIZE
509 md5.
append(constant_values);
510 md5.
append(source[pso_type]);
512 md5.
append(
string_printf(
"metalrt_features=%llu", kernel_features & METALRT_FEATURE_MASK));
514 if (pso_type != PSO_GENERIC) {
518 kernels_md5[pso_type] = md5.
get_hex();
521void MetalDevice::compile_and_load(
const int device_id, MetalPipelineType pso_type)
527 id<MTLDevice> mtlDevice;
535 MetalDevice *instance = get_device_by_ID(device_id,
lock);
537 metal_printf(
"Ignoring %s compilation request - device no longer exists",
538 kernel_type_as_string(pso_type));
542 if (!MetalDeviceKernels::should_load_kernels(instance, pso_type)) {
545 metal_printf(
"Ignoreing %s compilation request - kernels already requested",
546 kernel_type_as_string(pso_type));
550 mtlDevice = instance->mtlDevice;
551 source = instance->source[pso_type];
557 MTLCompileOptions *
options = [[MTLCompileOptions alloc]
init];
560 options.languageVersion = MTLLanguageVersion3_0;
561# if defined(MAC_OS_VERSION_14_0)
562 if (@available(macos 14.0, *)) {
563 options.languageVersion = MTLLanguageVersion3_1;
566# if defined(MAC_OS_VERSION_15_0)
567 if (@available(macos 15.0, *)) {
568 options.languageVersion = MTLLanguageVersion3_2;
569 if (
const char *loglevel = getenv(
"MTL_LOG_LEVEL")) {
570 if (strcmp(loglevel,
"MTLLogLevelDebug") == 0) {
577 if (getenv(
"CYCLES_METAL_PROFILING") || getenv(
"CYCLES_METAL_DEBUG")) {
584 NSError *error =
nullptr;
585 id<MTLLibrary> mtlLibrary = [mtlDevice newLibraryWithSource:@(source.c_str())
589 metal_printf(
"Front-end compilation finished in %.1f seconds (%s)",
591 kernel_type_as_string(pso_type));
595 bool blocking_pso_build = (getenv(
"CYCLES_METAL_PROFILING") ||
596 MetalDeviceKernels::is_benchmark_warmup());
597 if (blocking_pso_build) {
598 MetalDeviceKernels::wait_for_all();
606 if (MetalDevice *instance = get_device_by_ID(device_id,
lock)) {
608 if (error && [error localizedDescription]) {
610 << [[error localizedDescription] UTF8String];
613 instance->mtlLibrary[pso_type] = mtlLibrary;
616 MetalDeviceKernels::load(instance, pso_type);
619 NSString *err = [error localizedDescription];
620 instance->set_error(
string_printf(
"Failed to compile library:\n%s", [err UTF8String]));
625 if (starttime && blocking_pso_build) {
626 MetalDeviceKernels::wait_for_all();
628 metal_printf(
"Back-end compilation finished in %.1f seconds (%s)",
630 kernel_type_as_string(pso_type));
645 auto it = metal_mem_map.find(&mem);
646 if (it != metal_mem_map.end()) {
647 MetalMem *mmem = it->second.get();
650 if (mmem->pointer_index >= 0) {
652 pointers[mmem->pointer_index] = 0;
654 metal_mem_map.erase(it);
658bool MetalDevice::max_working_set_exceeded(
const size_t safety_margin)
const
662 size_t available = [mtlDevice recommendedMaxWorkingSetSize] - safety_margin;
663 return (stats.
mem_used > available);
666MetalDevice::MetalMem *MetalDevice::generic_alloc(
device_memory &mem)
673 id<MTLBuffer> metal_buffer = nil;
674 MTLResourceOptions
options = MTLResourceStorageModeShared;
678 options = MTLResourceStorageModePrivate;
684 set_error(
"System is out of GPU memory");
694 metal_mem_alloc(metal_buffer);
696 metal_buffer.label = [NSString stringWithFormat:
@"%s", mem.
log_name().c_str()];
698 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
700 assert(metal_mem_map.count(&mem) == 0);
704 mmem->mtlBuffer = metal_buffer;
707 if (
options != MTLResourceStorageModePrivate) {
708 mmem->hostPtr = [metal_buffer contents];
711 mmem->hostPtr =
nullptr;
718 if (metal_buffer.storageMode == MTLStorageModeShared) {
730 MetalMem *mmem_ptr = mmem.get();
731 metal_mem_map[&mem] = std::move(mmem);
733 if (max_working_set_exceeded()) {
734 set_error(
"System is out of GPU memory");
758 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
759 MetalMem &mmem = *metal_mem_map.at(&mem);
760 size_t size = mmem.size;
762 bool free_mtlBuffer =
true;
768 free_mtlBuffer =
false;
772 if (free_mtlBuffer) {
777 assert(!
"Metal device should not copy memory back to host");
785 metal_mem_free(mmem.mtlBuffer);
786 mmem.mtlBuffer = nil;
789 erase_allocation(mem);
795 assert(!
"mem_alloc not supported for images.");
816 generic_copy_to(mem);
821 generic_copy_to(mem);
827 generic_copy_to(mem);
835 assert(!
"Metal does not support mem_move_to_host");
838void MetalDevice::mem_copy_from(
839 device_memory & ,
const size_t ,
size_t ,
const size_t ,
size_t )
880void MetalDevice::cancel()
886 active_device_ids.erase(device_id);
891bool MetalDevice::is_ready(
string &status)
const
893 if (!error_msg.empty()) {
898 int num_loaded = MetalDeviceKernels::get_loaded_kernel_count(
this, PSO_GENERIC);
900 status =
string_printf(
"%d / %d render kernels loaded (may take a few minutes the first time)",
906 if (
int num_requests = MetalDeviceKernels::num_incomplete_specialization_requests()) {
907 status =
string_printf(
"%d kernels to optimize", num_requests);
909 else if (kernel_specialization_level == PSO_SPECIALIZED_INTERSECT) {
910 status =
"Using optimized intersection kernels";
912 else if (kernel_specialization_level == PSO_SPECIALIZED_SHADE) {
913 status =
"Using optimized kernels";
916 metal_printf(
"MetalDevice::is_ready(...) --> true");
920bool MetalDevice::set_bvh_limits(
size_t instance_count,
size_t max_prim_count)
928 const int standard_limits_max_prim_count = (1 << 28);
929 const int standard_limits_max_instance_count = (1 << 24);
931 bool using_metalrt_extended_limits_before = use_metalrt_extended_limits;
935 if (instance_count > standard_limits_max_instance_count ||
936 max_prim_count > standard_limits_max_prim_count)
938 use_metalrt_extended_limits =
true;
939 metal_printf(
"Enabling MetalRT extended limits (max_prim_count = %zu, instance_count = %zu)",
945 return using_metalrt_extended_limits_before != use_metalrt_extended_limits;
948void MetalDevice::optimize_for_scene(
Scene *scene)
950 MetalPipelineType specialization_level = kernel_specialization_level;
954 specialization_level = (MetalPipelineType)
min(specialization_level, PSO_SPECIALIZED_INTERSECT);
959 int this_device_id = this->device_id;
960 auto specialize_kernels_fn = ^() {
961 for (
int level = 1; level <= int(specialization_level); level++) {
962 compile_and_load(this_device_id, MetalPipelineType(level));
967 bool specialize_in_background =
true;
970 if (getenv(
"CYCLES_METAL_PROFILING") !=
nullptr) {
971 specialize_in_background =
false;
975 if (MetalDeviceKernels::is_benchmark_warmup()) {
976 specialize_in_background =
false;
979 if (specialize_in_background) {
980 if (MetalDeviceKernels::num_incomplete_specialization_requests() == 0) {
981 dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0),
982 specialize_kernels_fn);
985 metal_printf(
"\"optimize_for_scene\" request already in flight - dropping request");
989 specialize_kernels_fn();
993void MetalDevice::const_copy_to(
const char *name,
void *host,
const size_t size)
995 if (strcmp(name,
"data") == 0) {
997 memcpy((uint8_t *)&launch_params->data, host,
sizeof(KernelData));
1000 for (
int level = 1; level <= int(kernel_specialization_level); level++) {
1001 refresh_source_and_kernels_md5(MetalPipelineType(level));
1006 auto update_launch_pointers = [&](
size_t offset,
void *
data,
const size_t pointers_size) {
1009 MetalMem **mmem = (MetalMem **)data;
1010 int pointer_count = pointers_size /
sizeof(
device_ptr);
1011 int pointer_index = offset /
sizeof(
device_ptr);
1012 for (
int i = 0; i < pointer_count; i++) {
1015 mmem[i]->pointer_index = pointer_index + i;
1016 if (mmem[i]->mtlBuffer) {
1017 addresses[i] = mmem[i]->mtlBuffer.gpuAddress;
1024 if (strcmp(name,
"integrator_state") == 0) {
1027 update_launch_pointers(
1032 memcpy((uint8_t *)&launch_params->integrator_state + pointer_block_size,
1033 (uint8_t *)host + pointer_block_size,
1036# define KERNEL_DATA_ARRAY(data_type, tex_name) \
1037 else if (strcmp(name, #tex_name) == 0) \
1039 update_launch_pointers(offsetof(KernelParamsMetal, tex_name), host, size); \
1042# undef KERNEL_DATA_ARRAY
1049 generic_copy_to(mem);
1062void MetalDevice::image_alloc_as_buffer(
device_image &mem)
1064 MetalDevice::MetalMem *mmem = generic_alloc(mem);
1065 generic_copy_to(mem);
1067 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
1072 if (image_info_id >= image_info.size()) {
1074 const size_t new_size =
max(
size_t(image_info_id) + 128, image_info.size() * 2);
1075 image_info.host_only_resize(new_size);
1076 image_info_id_map.resize(new_size);
1079 image_info[image_info_id] = mem.
info;
1080 image_info_id_map[image_info_id] = mmem->mtlBuffer;
1083 using_nanovdb =
true;
1096 "Texture exceeds maximum allowed size of 16384 x 16384 (requested: %zu x %zu)",
1116 MTLPixelFormat formats[] = {MTLPixelFormatR8Unorm,
1117 MTLPixelFormatRG8Unorm,
1118 MTLPixelFormatInvalid,
1119 MTLPixelFormatRGBA8Unorm};
1123 MTLPixelFormat formats[] = {MTLPixelFormatR16Unorm,
1124 MTLPixelFormatRG16Unorm,
1125 MTLPixelFormatInvalid,
1126 MTLPixelFormatRGBA16Unorm};
1130 MTLPixelFormat formats[] = {MTLPixelFormatR32Uint,
1131 MTLPixelFormatRG32Uint,
1132 MTLPixelFormatInvalid,
1133 MTLPixelFormatRGBA32Uint};
1137 MTLPixelFormat formats[] = {MTLPixelFormatR32Sint,
1138 MTLPixelFormatRG32Sint,
1139 MTLPixelFormatInvalid,
1140 MTLPixelFormatRGBA32Sint};
1144 MTLPixelFormat formats[] = {MTLPixelFormatR32Float,
1145 MTLPixelFormatRG32Float,
1146 MTLPixelFormatInvalid,
1147 MTLPixelFormatRGBA32Float};
1151 MTLPixelFormat formats[] = {MTLPixelFormatR16Float,
1152 MTLPixelFormatRG16Float,
1153 MTLPixelFormatInvalid,
1154 MTLPixelFormatRGBA16Float};
1162 assert(format != MTLPixelFormatInvalid);
1164 id<MTLTexture> mtlTexture = nil;
1169 MTLTextureDescriptor *desc;
1171 desc = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:
format
1176 desc.storageMode = MTLStorageModeShared;
1177 desc.usage = MTLTextureUsageShaderRead;
1182 desc.allowGPUOptimizedContents =
false;
1188 mtlTexture = [mtlDevice newTextureWithDescriptor:desc];
1190 set_error(
"System is out of GPU memory");
1196 withBytes:mem.host_pointer
1197 bytesPerRow:src_pitch];
1201 image_alloc_as_buffer(mem);
1207 metal_mem_alloc(mtlTexture);
1209 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
1212 mmem->mtlTexture = mtlTexture;
1213 metal_mem_map[&mem] = std::move(mmem);
1218 if (image_info_id >= image_info.size()) {
1220 const size_t new_size =
max(
size_t(image_info_id) + 128, image_info.size() * 2);
1221 image_info.host_only_resize(new_size);
1222 image_info_id_map.resize(new_size);
1226 image_info_id_map[image_info_id] = mtlTexture;
1227 image_info[image_info_id] = mem.
info;
1228 image_info[image_info_id].
data =
uint64_t(image_info_id) | (sampler_index << 32);
1230 if (max_working_set_exceeded()) {
1231 set_error(
"System is out of GPU memory");
1242 id<MTLTexture> mtlTexture;
1244 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
1245 mtlTexture = metal_mem_map.at(&mem)->mtlTexture;
1249 withBytes:mem.host_pointer
1250 bytesPerRow:src_pitch];
1253 generic_copy_to(mem);
1264 else if (metal_mem_map.contains(&mem)) {
1265 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
1266 MetalMem &mmem = *metal_mem_map.at(&mem);
1269 metal_mem_free(mmem.mtlTexture);
1270 mmem.mtlTexture = nil;
1271 erase_allocation(mem);
1273 image_info_id_map[image_info_id] = nil;
1278 return make_unique<MetalDeviceQueue>(
this);
1288void *MetalDevice::get_native_buffer(
device_ptr ptr)
1290 return ((MetalMem *)ptr)->mtlBuffer;
1293void MetalDevice::flush_delayed_free_list()
1298 std::lock_guard<std::recursive_mutex>
lock(metal_mem_map_mutex);
1299 for (
auto &it : delayed_free_list) {
1302 delayed_free_list.clear();
1313 BVHMetal *bvh_metal =
static_cast<BVHMetal *
>(bvh);
1314 bvh_metal->motion_blur = motion_blur;
1315 bvh_metal->use_pcmi = use_pcmi;
1316 bvh_metal->extended_limits = use_metalrt_extended_limits;
1317 if (bvh_metal->build(progress, mtlDevice, mtlGeneralCommandQueue,
refit)) {
1320 update_bvh(bvh_metal);
1324 if (max_working_set_exceeded()) {
1325 set_error(
"System is out of GPU memory");
1330void MetalDevice::free_bvh()
1334 for (id<MTLAccelerationStructure> &blas : unique_blas_array) {
1335 metal_mem_free(blas);
1337 unique_blas_array.clear();
1341 metal_mem_free(blas_buffer);
1346 metal_mem_free(accel_struct);
1351void MetalDevice::update_bvh(BVHMetal *bvh_metal)
1359 accel_struct = bvh_metal->accel_struct;
1360 unique_blas_array = bvh_metal->unique_blas_array;
1361 blas_array = bvh_metal->blas_array;
1365 metal_mem_alloc(accel_struct);
1367 [accel_struct retain];
1368 for (id<MTLAccelerationStructure> &blas : unique_blas_array) {
1370 metal_mem_alloc(blas);
1375 blas_buffer = [mtlDevice newBufferWithLength:buffer_size
options:MTLResourceStorageModeShared];
1376 metal_mem_alloc(blas_buffer);
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
void refit(btStridingMeshInterface *triangles, const btVector3 &aabbMin, const btVector3 &aabbMax)
KernelOptimizationLevel kernel_optimization_level
bool use_hardware_raytracing
virtual void build_bvh(BVH *bvh, Progress &progress, bool refit)
virtual void set_error(const string &error)
void append(const uint8_t *data, const int nbytes)
void mem_alloc(const size_t size)
void mem_free(const size_t size)
string log_name() const override
bool is_resident(Device *sub_device) const
void * host_alloc(const size_t size)
const char * global_name() const
device_ptr device_pointer
virtual string log_name() const
CCL_NAMESPACE_BEGIN struct Options options
#define KERNEL_FEATURE_OBJECT_MOTION
#define CCL_NAMESPACE_END
@ KERNEL_OPTIMIZATION_LEVEL_OFF
@ KERNEL_OPTIMIZATION_LEVEL_FULL
@ KERNEL_OPTIMIZATION_LEVEL_INTERSECT
#define assert(assertion)
BMesh const char void * data
std::string format(const std::string &fmt, Args &&...args)
string path_cache_get(const string &sub)
string path_source_replace_includes(const string &source, const string &path)
string path_get(const string &sub)
bool path_write_text(const string &path, const string &text)
string string_human_readable_size(size_t size)
string to_string(const char *str)
string string_human_readable_number(size_t num)
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
void string_replace_same_length(string &haystack, const string &needle, const string &other)
std::unique_lock< std::mutex > thread_scoped_lock
CCL_NAMESPACE_BEGIN double time_dt()
ccl_device_inline bool is_nanovdb_type(int type)