24MetalDeviceQueue::MetalDeviceQueue(MetalDevice *device)
25 :
DeviceQueue(device), metal_device_(device), stats_(device->stats)
28 command_buffer_desc_ = [[MTLCommandBufferDescriptor alloc]
init];
29 command_buffer_desc_.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;
31 mtlDevice_ = device->mtlDevice;
32 mtlCommandQueue_ = device->mtlComputeCommandQueue;
34 shared_event_ = [mtlDevice_ newSharedEvent];
38 event_queue_ = dispatch_queue_create(
"com.cycles.metal.event_queue",
nullptr);
39 shared_event_listener_ = [[MTLSharedEventListener alloc] initWithDispatchQueue:event_queue_];
41 wait_semaphore_ = dispatch_semaphore_create(0);
43 if (
auto *
str = getenv(
"CYCLES_METAL_PROFILING")) {
44 if (atoi(
str) && [mtlDevice_ supportsCounterSampling:MTLCounterSamplingPointAtStageBoundary])
47 profiling_enabled_ =
true;
48 label_command_encoders_ =
true;
51 if (getenv(
"CYCLES_METAL_DEBUG")) {
53 verbose_tracing_ =
true;
54 label_command_encoders_ =
true;
61void MetalDeviceQueue::setup_capture()
65 if (
auto *capture_kernel_str = getenv(
"CYCLES_DEBUG_METAL_CAPTURE_KERNEL")) {
67 capture_kernel_ =
DeviceKernel(atoi(capture_kernel_str));
70 capture_dispatch_counter_ = 0;
71 if (
auto *capture_dispatch_str = getenv(
"CYCLES_DEBUG_METAL_CAPTURE_DISPATCH")) {
72 capture_dispatch_counter_ = atoi(capture_dispatch_str);
74 printf(
"Capture dispatch number %d\n", capture_dispatch_counter_);
77 else if (
auto *capture_samples_str = getenv(
"CYCLES_DEBUG_METAL_CAPTURE_SAMPLES")) {
80 capture_samples_ =
true;
81 capture_reset_counter_ = atoi(capture_samples_str);
83 capture_dispatch_counter_ =
INT_MAX;
84 if (
auto *capture_limit_str = getenv(
"CYCLES_DEBUG_METAL_CAPTURE_LIMIT")) {
86 capture_dispatch_counter_ = atoi(capture_limit_str);
89 printf(
"Capturing sample block %d (dispatch limit: %d)\n",
90 capture_reset_counter_,
91 capture_dispatch_counter_);
99 MTLCaptureManager *captureManager = [MTLCaptureManager sharedCaptureManager];
100 mtlCaptureScope_ = [captureManager newCaptureScopeWithDevice:mtlDevice_];
101 mtlCaptureScope_.label = [NSString stringWithFormat:
@"Cycles kernel dispatch"];
102 [captureManager setDefaultCaptureScope:mtlCaptureScope_];
104 label_command_encoders_ =
true;
106 if (
auto *capture_url = getenv(
"CYCLES_DEBUG_METAL_CAPTURE_URL")) {
107 if ([captureManager supportsDestination:MTLCaptureDestinationGPUTraceDocument]) {
109 MTLCaptureDescriptor *captureDescriptor = [[MTLCaptureDescriptor alloc]
init];
110 captureDescriptor.captureObject = mtlCaptureScope_;
111 captureDescriptor.destination = MTLCaptureDestinationGPUTraceDocument;
112 captureDescriptor.outputURL = [NSURL fileURLWithPath:@(capture_url)];
115 if (![captureManager startCaptureWithDescriptor:captureDescriptor error:&error]) {
116 NSString *err = [error localizedDescription];
117 printf(
"Start capture failed: %s\n", [err UTF8String]);
120 printf(
"Capture started (URL: %s)\n", capture_url);
121 is_capturing_to_disk_ =
true;
125 printf(
"Capture to file is not supported\n");
130void MetalDeviceQueue::update_capture(
DeviceKernel kernel)
134 capture_dispatch_counter_ -= 1;
142 if (capture_dispatch_counter_ < 0) {
148 if (kernel == capture_kernel_) {
150 if (capture_dispatch_counter_ == 0) {
153 capture_dispatch_counter_ -= 1;
158 if (capture_samples_) {
160 if (capture_reset_counter_ == 0) {
165 capture_reset_counter_ -= 1;
171void MetalDeviceQueue::begin_capture()
174 if (mtlCommandBuffer_) {
177 [mtlCaptureScope_ beginScope];
178 printf(
"[mtlCaptureScope_ beginScope]\n");
179 is_capturing_ =
true;
182void MetalDeviceQueue::end_capture()
184 [mtlCaptureScope_ endScope];
185 is_capturing_ =
false;
186 printf(
"[mtlCaptureScope_ endScope]\n");
188 if (is_capturing_to_disk_) {
189 [[MTLCaptureManager sharedCaptureManager] stopCapture];
190 has_captured_to_disk_ =
true;
191 is_capturing_to_disk_ =
false;
192 is_capturing_ =
false;
193 printf(
"Capture stopped\n");
197MetalDeviceQueue::~MetalDeviceQueue()
201 assert(mtlCommandBuffer_ == nil);
202 assert(command_buffers_submitted_ == command_buffers_completed_);
204 close_compute_encoder();
205 close_blit_encoder();
207 [shared_event_listener_ release];
208 [shared_event_ release];
209 [command_buffer_desc_ release];
211 if (mtlCaptureScope_) {
212 [mtlCaptureScope_ release];
215 double total_time = 0.0;
219 int64_t num_pathtracing_dispatches = 0;
221 auto &stat = timing_stats_[i];
224 total_time += stat.total_time;
225 num_dispatches += stat.num_dispatches;
226 num_pathtracing_dispatches += pathtracing_kernel ? stat.num_dispatches : 0;
228 bool has_extra = (num_pathtracing_dispatches && num_dispatches > num_pathtracing_dispatches);
230 if (num_dispatches) {
231 printf(
"\nMetal %sdispatch stats:\n", num_pathtracing_dispatches ?
"path-tracing " :
"");
239 auto divider = string(header.length(),
'-');
240 printf(
"%s\n%s\n%s\n", divider.c_str(), header.c_str(), divider.c_str());
243 auto &stat = timing_stats_[i];
247 if ((pathtracing_kernel && num_pathtracing_dispatches) || stat.num_dispatches > 0) {
248 printf(
"%-40s %16llu %12llu %12llu %9.4f %9.2f\n",
250 stat.total_work_size,
252 stat.total_work_size / stat.num_dispatches,
254 stat.total_time * 100.0 / total_time);
257 printf(
"%s\n", divider.c_str());
260 printf(
"%s\n", divider.c_str());
261 printf(
"%-40s %16s %12llu %12s %9.4f %9.2f\n",
"",
"", num_dispatches,
"", total_time, 100.0);
262 printf(
"%s\n\n", divider.c_str());
272 if (MetalInfo::get_apple_gpu_architecture(metal_device_->mtlDevice) == APPLE_M1) {
284void MetalDeviceQueue::get_memory_info(
size_t &total,
size_t &free)
const
286 total = [metal_device_->mtlDevice recommendedMaxWorkingSetSize];
287 free = total - std::min(stats_.mem_used, total);
290int MetalDeviceQueue::num_sort_partitions(
int max_num_paths, uint max_scene_shaders)
const
292 int sort_partition_elements = MetalInfo::optimal_sort_partition_elements();
297 if (max_scene_shaders < 300 && sort_partition_elements > 0) {
298 return max(max_num_paths / sort_partition_elements, 1);
305bool MetalDeviceQueue::supports_local_atomic_sort()
const
307 return metal_device_->use_local_atomic_sort();
310static void zero_resource(
void *address_in_arg_buffer,
int index = 0)
316template<
class T>
void write_resource(
void *address_in_arg_buffer,
T resource,
int index = 0)
318 zero_resource(address_in_arg_buffer,
index);
321 MTLResourceID resource_id = resource.gpuResourceID;
326template<>
void write_resource(
void *address_in_arg_buffer, id<MTLBuffer> buffer,
int index)
328 zero_resource(address_in_arg_buffer,
index);
331 pptr[
index] = buffer.gpuAddress;
335static id<MTLBuffer> patch_resource(
void *address_in_arg_buffer,
int index = 0)
338 if (MetalDevice::MetalMem *mmem = (MetalDevice::MetalMem *)pptr[
index]) {
339 write_resource<id<MTLBuffer>>(address_in_arg_buffer, mmem->mtlBuffer,
index);
340 return mmem->mtlBuffer;
345void MetalDeviceQueue::init_execution()
349 for (
uint64_t slot = 0; slot < metal_device_->blas_array.size(); ++slot) {
350 write_resource(blas_array, metal_device_->blas_array[slot], slot);
360void MetalDeviceQueue::load_image_info()
370 std::lock_guard<std::recursive_mutex>
lock(metal_device_->metal_mem_map_mutex);
373 id<MTLBuffer> &image_bindings = metal_device_->image_bindings;
374 std::vector<id<MTLResource>> &image_info_id_map = metal_device_->image_info_id_map;
378 metal_device_->generic_free(image_info);
381 const size_t min_bindings_length =
sizeof(
void *) * image_info.
size();
382 if (image_bindings.length < min_bindings_length) {
383 metal_device_->metal_mem_free(image_bindings);
384 image_bindings = [metal_device_->mtlDevice newBufferWithLength:min_bindings_length
385 options:MTLResourceStorageModeShared];
386 metal_device_->metal_mem_alloc(image_bindings);
394 memset(bindings, 0, image_bindings.length);
395 for (
int image_info_id = 0; image_info_id < image_info.
size(); ++image_info_id) {
396 if (image_info_id_map[image_info_id]) {
397 if (metal_device_->is_texture(image_info[image_info_id])) {
398 write_resource(bindings, id<MTLTexture>(image_info_id_map[image_info_id]), image_info_id);
403 &image_info[image_info_id].data, id<MTLBuffer>(image_info_id_map[image_info_id]), 0);
414 update_capture(kernel);
416 if (metal_device_->have_error()) {
425 id<MTLComputeCommandEncoder> mtlComputeCommandEncoder = get_compute_encoder(kernel);
427 if (profiling_enabled_) {
428 command_encoder_labels_.push_back({kernel,
work_size, current_encoder_idx_});
430 if (label_command_encoders_) {
432 mtlComputeCommandEncoder.label = [NSString
433 stringWithFormat:
@"Metal queue launch %s, work_size %d",
438 if (!active_pipelines_[kernel].update(metal_device_, kernel)) {
439 metal_device_->set_error(
443 MetalDispatchPipeline &active_pipeline = active_pipelines_[kernel];
445 uint8_t dynamic_args[512] = {0};
448 size_t dynamic_bytes_written = 0;
449 size_t max_size_in_bytes = 0;
450 for (
size_t i = 0; i < args.
count; i++) {
451 size_t size_in_bytes = args.
sizes[i];
452 max_size_in_bytes =
max(max_size_in_bytes, size_in_bytes);
453 dynamic_bytes_written =
round_up(dynamic_bytes_written, size_in_bytes);
454 memcpy(dynamic_args + dynamic_bytes_written, args.
values[i], size_in_bytes);
456 patch_resource(dynamic_args + dynamic_bytes_written);
458 dynamic_bytes_written += size_in_bytes;
462 dynamic_bytes_written =
round_up(dynamic_bytes_written, max_size_in_bytes);
465 assert(dynamic_bytes_written <=
sizeof(dynamic_args));
467 uint64_t ancillary_args[ANCILLARY_SLOT_COUNT] = {0};
470 int ancillary_index = 0;
471 write_resource(ancillary_args, metal_device_->image_bindings, ancillary_index++);
473 if (metal_device_->use_metalrt) {
474 write_resource(ancillary_args, metal_device_->accel_struct, ancillary_index++);
475 write_resource(ancillary_args, metal_device_->blas_buffer, ancillary_index++);
478 for (
int table_idx = 0; table_idx < METALRT_TABLE_NUM; table_idx++) {
480 ancillary_args, active_pipeline.intersection_func_table[table_idx], ancillary_index++);
482 assert(ancillary_index == ANCILLARY_SLOT_COUNT);
485 [mtlComputeCommandEncoder setBytes:dynamic_args
length:dynamic_bytes_written atIndex:0];
486 [mtlComputeCommandEncoder setBuffer:metal_device_->launch_params_buffer offset:0 atIndex:1];
487 [mtlComputeCommandEncoder setBytes:ancillary_args
length:
sizeof(ancillary_args) atIndex:2];
491 if (!metal_device_->mtlResidencySet_enabled && metal_device_->use_metalrt &&
494 if (id<MTLAccelerationStructure> accel_struct = metal_device_->accel_struct) {
496 [mtlComputeCommandEncoder useResource:accel_struct usage:MTLResourceUsageRead];
497 if (metal_device_->blas_buffer) {
498 [mtlComputeCommandEncoder useResource:metal_device_->blas_buffer
499 usage:MTLResourceUsageRead];
501 [mtlComputeCommandEncoder useResources:metal_device_->unique_blas_array.data()
502 count:metal_device_->unique_blas_array.size()
503 usage:MTLResourceUsageRead];
506 for (
int table = 0; table < METALRT_TABLE_NUM; table++) {
507 if (active_pipeline.intersection_func_table[table]) {
508 [mtlComputeCommandEncoder useResource:active_pipeline.intersection_func_table[table]
509 usage:MTLResourceUsageRead];
514 [mtlComputeCommandEncoder setComputePipelineState:active_pipeline.pipeline];
517 const int num_threads_per_block = active_pipeline.num_threads_per_block;
519 int shared_mem_bytes = 0;
532 shared_mem_bytes = (int)
round_up((num_threads_per_block + 1) *
sizeof(
int), 16);
537 int key_count = metal_device_->launch_params->data.max_shaders;
538 shared_mem_bytes = (int)
round_up(key_count *
sizeof(
int), 16);
546 if (shared_mem_bytes) {
547 assert(shared_mem_bytes <= 32 * 1024);
548 [mtlComputeCommandEncoder setThreadgroupMemoryLength:shared_mem_bytes atIndex:0];
551 MTLSize size_threads_per_dispatch = MTLSizeMake(
work_size, 1, 1);
552 MTLSize size_threads_per_threadgroup = MTLSizeMake(num_threads_per_block, 1, 1);
553 [mtlComputeCommandEncoder dispatchThreads:size_threads_per_dispatch
554 threadsPerThreadgroup:size_threads_per_threadgroup];
556 metal_device_->prepare_residency();
558 [mtlCommandBuffer_ addCompletedHandler:^(id<MTLCommandBuffer> command_buffer) {
561 if (command_buffer.status != MTLCommandBufferStatusCompleted) {
563 int(command_buffer.status));
565 if (command_buffer.error) {
567 const char *errCStr = [[NSString stringWithFormat:
@"%@", command_buffer.error]
570 kernel_type_as_string(active_pipeline.pso_type),
576 metal_device_->set_error(
str);
580 if (verbose_tracing_ || is_capturing_) {
585 if (verbose_tracing_) {
588 "_____________________________________.____________________.______________._________"
590 "______________________________________\n");
593 printf(
"%-40s| %7d threads |%5.2fms | buckets [",
596 last_completion_time_ * 1000.0);
597 std::lock_guard<std::recursive_mutex>
lock(metal_device_->metal_mem_map_mutex);
598 for (
auto &it : metal_device_->metal_mem_map) {
599 const string c_integrator_queue_counter =
"integrator_queue_counter";
600 if (it.first->global_name() == c_integrator_queue_counter) {
602 it.first->host_pointer)
605 printf(
"%s%d", i == 0 ?
"" :
",", queue_counter->num_queued[i]);
617 return !(metal_device_->have_error());
621void MetalDeviceQueue::flush_timing_stats()
623 for (
auto label : command_encoder_labels_) {
624 TimingStats &stat = timing_stats_[label.kernel];
626 double completion_time_gpu;
627 NSData *computeTimeStamps = [metal_device_->mtlCounterSampleBuffer
628 resolveCounterRange:NSMakeRange(label.timing_id, 2)];
629 MTLCounterResultTimestamp *timestamps = (MTLCounterResultTimestamp *)(computeTimeStamps.bytes);
631 uint64_t begTime = timestamps[0].timestamp;
632 uint64_t endTime = timestamps[1].timestamp;
633 completion_time_gpu = (endTime - begTime) / (
double)NSEC_PER_SEC;
635 stat.num_dispatches++;
636 stat.total_time += completion_time_gpu;
637 stat.total_work_size += label.work_size;
638 last_completion_time_ = completion_time_gpu;
640 command_encoder_labels_.clear();
643bool MetalDeviceQueue::synchronize()
646 if (has_captured_to_disk_ || metal_device_->have_error()) {
650 close_compute_encoder();
651 close_blit_encoder();
653 if (mtlCommandBuffer_) {
656 uint64_t shared_event_id_ = this->shared_event_id_++;
658 __block dispatch_semaphore_t block_sema = wait_semaphore_;
659 [shared_event_ notifyListener:shared_event_listener_
660 atValue:shared_event_id_
661 block:^(id<MTLSharedEvent> ,
uint64_t ) {
662 dispatch_semaphore_signal(block_sema);
665 [mtlCommandBuffer_ encodeSignalEvent:shared_event_ value:shared_event_id_];
666 [mtlCommandBuffer_ commit];
667 dispatch_semaphore_wait(wait_semaphore_, DISPATCH_TIME_FOREVER);
669 [mtlCommandBuffer_ release];
671 metal_device_->flush_delayed_free_list();
673 mtlCommandBuffer_ = nil;
674 flush_timing_stats();
679 return !(metal_device_->have_error());
686 if (metal_device_->have_error()) {
698 metal_device_->mem_alloc(mem);
704 std::lock_guard<std::recursive_mutex>
lock(metal_device_->metal_mem_map_mutex);
705 MetalDevice::MetalMem &mmem = *metal_device_->metal_mem_map.at(&mem);
706 if (mmem.mtlBuffer) {
707 id<MTLBlitCommandEncoder> blitEncoder = get_blit_encoder();
708 [blitEncoder fillBuffer:mmem.mtlBuffer range:NSMakeRange(mmem.offset, mmem.size) value:0];
711 metal_device_->mem_zero(mem);
719 if (metal_device_->have_error()) {
729 metal_device_->mem_alloc(mem);
743void *MetalDeviceQueue::copy_from_device_synchronized(
device_memory &mem,
754 return (d_ptr) ?
reinterpret_cast<MetalDevice::MetalMem *
>(d_ptr)->hostPtr :
nullptr;
757void MetalDeviceQueue::prepare_resources()
759 if (metal_device_->mtlResidencySet_enabled) {
764 std::lock_guard<std::recursive_mutex>
lock(metal_device_->metal_mem_map_mutex);
767 for (
auto &it : metal_device_->metal_mem_map) {
770 MTLResourceUsage usage = MTLResourceUsageRead;
772 usage |= MTLResourceUsageWrite;
775 if (it.second->mtlBuffer) {
777 [mtlComputeEncoder_ useResource:it.second->mtlBuffer usage:usage];
779 else if (it.second->mtlTexture) {
781 [mtlComputeEncoder_ useResource:it.second->mtlTexture usage:usage];
786 [mtlComputeEncoder_ useResource:metal_device_->image_bindings usage:MTLResourceUsageRead];
789id<MTLComputeCommandEncoder> MetalDeviceQueue::get_compute_encoder(
DeviceKernel kernel)
793 if (profiling_enabled_) {
795 close_compute_encoder();
798 if (mtlComputeEncoder_) {
799 if (mtlComputeEncoder_.dispatchType == concurrent ? MTLDispatchTypeConcurrent :
800 MTLDispatchTypeSerial)
805 return mtlComputeEncoder_;
807 close_compute_encoder();
810 close_blit_encoder();
812 if (!mtlCommandBuffer_) {
813 mtlCommandBuffer_ = [mtlCommandQueue_ commandBuffer];
814 [mtlCommandBuffer_ retain];
817 if (profiling_enabled_) {
818 MTLComputePassDescriptor *desc = [[MTLComputePassDescriptor alloc]
init];
820 current_encoder_idx_ = (counter_sample_buffer_curr_idx_.fetch_add(2) %
821 MAX_SAMPLE_BUFFER_LENGTH);
822 [desc.sampleBufferAttachments[0] setSampleBuffer:metal_device_->mtlCounterSampleBuffer];
823 [desc.sampleBufferAttachments[0] setStartOfEncoderSampleIndex:current_encoder_idx_];
824 [desc.sampleBufferAttachments[0] setEndOfEncoderSampleIndex:current_encoder_idx_ + 1];
826 [desc setDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
828 mtlComputeEncoder_ = [mtlCommandBuffer_ computeCommandEncoderWithDescriptor:desc];
831 mtlComputeEncoder_ = [mtlCommandBuffer_
832 computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent :
833 MTLDispatchTypeSerial];
836 [mtlComputeEncoder_ retain];
842 return mtlComputeEncoder_;
845id<MTLBlitCommandEncoder> MetalDeviceQueue::get_blit_encoder()
847 if (mtlBlitEncoder_) {
848 return mtlBlitEncoder_;
851 close_compute_encoder();
853 if (!mtlCommandBuffer_) {
854 mtlCommandBuffer_ = [mtlCommandQueue_ commandBuffer];
855 [mtlCommandBuffer_ retain];
858 mtlBlitEncoder_ = [mtlCommandBuffer_ blitCommandEncoder];
859 [mtlBlitEncoder_ retain];
860 return mtlBlitEncoder_;
863void MetalDeviceQueue::close_compute_encoder()
865 if (mtlComputeEncoder_) {
866 [mtlComputeEncoder_ endEncoding];
867 [mtlComputeEncoder_ release];
868 mtlComputeEncoder_ = nil;
872void MetalDeviceQueue::close_blit_encoder()
874 if (mtlBlitEncoder_) {
875 [mtlBlitEncoder_ endEncoding];
876 [mtlBlitEncoder_ release];
877 mtlBlitEncoder_ = nil;
881void *MetalDeviceQueue::native_queue()
883 return mtlCommandQueue_;
888 return make_unique<MetalDeviceGraphicsInterop>(
this);
unsigned long long int uint64_t
virtual device_ptr mem_device_ptr(const device_memory &mem, Device *sub_device)
device_ptr device_pointer
CCL_NAMESPACE_BEGIN struct Options options
#define CCL_NAMESPACE_END
bool device_kernel_has_intersection(DeviceKernel kernel)
const char * device_kernel_as_string(DeviceKernel kernel)
#define assert(assertion)
float length(VecOp< float, D >) RET
ccl_gpu_kernel_postfix const ccl_global int ccl_global float const int work_size
@ DEVICE_GPU_KERNEL_INTEGRATOR_NUM
@ DEVICE_KERNEL_INTEGRATOR_RESET
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORTED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL
@ DEVICE_KERNEL_INTEGRATOR_ACTIVE_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_PATHS_ARRAY
void free(struct bMotionPath *mpath)
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
ccl_device_inline size_t round_up(const size_t x, const size_t multiple)