Blender V5.3
queue.mm
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#ifdef WITH_METAL
6
7# include <algorithm>
8# include <mutex>
9
10# include "device/metal/queue.h"
11
14# include "device/metal/kernel.h"
15
16# include "util/path.h"
17# include "util/string.h"
18# include "util/time.h"
19
21
22/* MetalDeviceQueue */
23
24MetalDeviceQueue::MetalDeviceQueue(MetalDevice *device)
25 : DeviceQueue(device), metal_device_(device), stats_(device->stats)
26{
27 @autoreleasepool {
28 command_buffer_desc_ = [[MTLCommandBufferDescriptor alloc] init];
29 command_buffer_desc_.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;
30
31 mtlDevice_ = device->mtlDevice;
32 mtlCommandQueue_ = device->mtlComputeCommandQueue;
33
34 shared_event_ = [mtlDevice_ newSharedEvent];
35 shared_event_id_ = 1;
36
37 /* Shareable event listener */
38 event_queue_ = dispatch_queue_create("com.cycles.metal.event_queue", nullptr);
39 shared_event_listener_ = [[MTLSharedEventListener alloc] initWithDispatchQueue:event_queue_];
40
41 wait_semaphore_ = dispatch_semaphore_create(0);
42
43 if (auto *str = getenv("CYCLES_METAL_PROFILING")) {
44 if (atoi(str) && [mtlDevice_ supportsCounterSampling:MTLCounterSamplingPointAtStageBoundary])
45 {
46 /* Enable per-kernel timing breakdown (shown at end of render). */
47 profiling_enabled_ = true;
48 label_command_encoders_ = true;
49 }
50 }
51 if (getenv("CYCLES_METAL_DEBUG")) {
52 /* Enable very verbose tracing (shows every dispatch). */
53 verbose_tracing_ = true;
54 label_command_encoders_ = true;
55 }
56
57 setup_capture();
58 }
59}
60
61void MetalDeviceQueue::setup_capture()
62{
63 capture_kernel_ = DeviceKernel(-1);
64
65 if (auto *capture_kernel_str = getenv("CYCLES_DEBUG_METAL_CAPTURE_KERNEL")) {
66 /* CYCLES_DEBUG_METAL_CAPTURE_KERNEL captures a single dispatch of the specified kernel. */
67 capture_kernel_ = DeviceKernel(atoi(capture_kernel_str));
68 printf("Capture kernel: %d = %s\n", capture_kernel_, device_kernel_as_string(capture_kernel_));
69
70 capture_dispatch_counter_ = 0;
71 if (auto *capture_dispatch_str = getenv("CYCLES_DEBUG_METAL_CAPTURE_DISPATCH")) {
72 capture_dispatch_counter_ = atoi(capture_dispatch_str);
73
74 printf("Capture dispatch number %d\n", capture_dispatch_counter_);
75 }
76 }
77 else if (auto *capture_samples_str = getenv("CYCLES_DEBUG_METAL_CAPTURE_SAMPLES")) {
78 /* CYCLES_DEBUG_METAL_CAPTURE_SAMPLES captures a block of dispatches from reset#(N) to
79 * reset#(N+1). */
80 capture_samples_ = true;
81 capture_reset_counter_ = atoi(capture_samples_str);
82
83 capture_dispatch_counter_ = INT_MAX;
84 if (auto *capture_limit_str = getenv("CYCLES_DEBUG_METAL_CAPTURE_LIMIT")) {
85 /* CYCLES_DEBUG_METAL_CAPTURE_LIMIT sets the maximum number of dispatches to capture. */
86 capture_dispatch_counter_ = atoi(capture_limit_str);
87 }
88
89 printf("Capturing sample block %d (dispatch limit: %d)\n",
90 capture_reset_counter_,
91 capture_dispatch_counter_);
92 }
93 else {
94 /* No capturing requested. */
95 return;
96 }
97
98 /* Enable .gputrace capture for the specified DeviceKernel. */
99 MTLCaptureManager *captureManager = [MTLCaptureManager sharedCaptureManager];
100 mtlCaptureScope_ = [captureManager newCaptureScopeWithDevice:mtlDevice_];
101 mtlCaptureScope_.label = [NSString stringWithFormat:@"Cycles kernel dispatch"];
102 [captureManager setDefaultCaptureScope:mtlCaptureScope_];
103
104 label_command_encoders_ = true;
105
106 if (auto *capture_url = getenv("CYCLES_DEBUG_METAL_CAPTURE_URL")) {
107 if ([captureManager supportsDestination:MTLCaptureDestinationGPUTraceDocument]) {
108
109 MTLCaptureDescriptor *captureDescriptor = [[MTLCaptureDescriptor alloc] init];
110 captureDescriptor.captureObject = mtlCaptureScope_;
111 captureDescriptor.destination = MTLCaptureDestinationGPUTraceDocument;
112 captureDescriptor.outputURL = [NSURL fileURLWithPath:@(capture_url)];
113
114 NSError *error;
115 if (![captureManager startCaptureWithDescriptor:captureDescriptor error:&error]) {
116 NSString *err = [error localizedDescription];
117 printf("Start capture failed: %s\n", [err UTF8String]);
118 }
119 else {
120 printf("Capture started (URL: %s)\n", capture_url);
121 is_capturing_to_disk_ = true;
122 }
123 }
124 else {
125 printf("Capture to file is not supported\n");
126 }
127 }
128}
129
130void MetalDeviceQueue::update_capture(DeviceKernel kernel)
131{
132 /* Handle capture end triggers. */
133 if (is_capturing_) {
134 capture_dispatch_counter_ -= 1;
135 if (capture_dispatch_counter_ <= 0 || kernel == DEVICE_KERNEL_INTEGRATOR_RESET) {
136 /* End capture if we've hit the dispatch limit or we hit a "reset". */
137 end_capture();
138 }
139 return;
140 }
141
142 if (capture_dispatch_counter_ < 0) {
143 /* We finished capturing. */
144 return;
145 }
146
147 /* Handle single-capture start trigger. */
148 if (kernel == capture_kernel_) {
149 /* Start capturing when we hit the Nth dispatch of the specified kernel. */
150 if (capture_dispatch_counter_ == 0) {
151 begin_capture();
152 }
153 capture_dispatch_counter_ -= 1;
154 return;
155 }
156
157 /* Handle multi-capture start trigger. */
158 if (capture_samples_) {
159 /* Start capturing when the reset countdown is at 0. */
160 if (capture_reset_counter_ == 0) {
161 begin_capture();
162 }
163
164 if (kernel == DEVICE_KERNEL_INTEGRATOR_RESET) {
165 capture_reset_counter_ -= 1;
166 }
167 return;
168 }
169}
170
171void MetalDeviceQueue::begin_capture()
172{
173 /* Start gputrace capture. */
174 if (mtlCommandBuffer_) {
175 synchronize();
176 }
177 [mtlCaptureScope_ beginScope];
178 printf("[mtlCaptureScope_ beginScope]\n");
179 is_capturing_ = true;
180}
181
182void MetalDeviceQueue::end_capture()
183{
184 [mtlCaptureScope_ endScope];
185 is_capturing_ = false;
186 printf("[mtlCaptureScope_ endScope]\n");
187
188 if (is_capturing_to_disk_) {
189 [[MTLCaptureManager sharedCaptureManager] stopCapture];
190 has_captured_to_disk_ = true;
191 is_capturing_to_disk_ = false;
192 is_capturing_ = false;
193 printf("Capture stopped\n");
194 }
195}
196
197MetalDeviceQueue::~MetalDeviceQueue()
198{
199 /* Tidying up here isn't really practical - we should expect and require the work
200 * queue to be empty here. */
201 assert(mtlCommandBuffer_ == nil);
202 assert(command_buffers_submitted_ == command_buffers_completed_);
203
204 close_compute_encoder();
205 close_blit_encoder();
206
207 [shared_event_listener_ release];
208 [shared_event_ release];
209 [command_buffer_desc_ release];
210
211 if (mtlCaptureScope_) {
212 [mtlCaptureScope_ release];
213 }
214
215 double total_time = 0.0;
216
217 /* Show per-kernel timings, if gathered (see CYCLES_METAL_PROFILING). */
218 int64_t num_dispatches = 0;
219 int64_t num_pathtracing_dispatches = 0;
220 for (size_t i = 0; i < DEVICE_KERNEL_NUM; i++) {
221 auto &stat = timing_stats_[i];
222 bool pathtracing_kernel = (i <= DEVICE_KERNEL_INTEGRATOR_RESET) &&
224 total_time += stat.total_time;
225 num_dispatches += stat.num_dispatches;
226 num_pathtracing_dispatches += pathtracing_kernel ? stat.num_dispatches : 0;
227 }
228 bool has_extra = (num_pathtracing_dispatches && num_dispatches > num_pathtracing_dispatches);
229
230 if (num_dispatches) {
231 printf("\nMetal %sdispatch stats:\n", num_pathtracing_dispatches ? "path-tracing " : "");
232 auto header = string_printf("%-40s %16s %12s %12s %9s %9s",
233 "Kernel name",
234 "Total threads",
235 "Dispatches",
236 "Avg. T/D",
237 "Time/s",
238 "Time/%");
239 auto divider = string(header.length(), '-');
240 printf("%s\n%s\n%s\n", divider.c_str(), header.c_str(), divider.c_str());
241
242 for (size_t i = 0; i < DEVICE_KERNEL_NUM; i++) {
243 auto &stat = timing_stats_[i];
244
245 bool pathtracing_kernel = (i <= DEVICE_KERNEL_INTEGRATOR_RESET) &&
247 if ((pathtracing_kernel && num_pathtracing_dispatches) || stat.num_dispatches > 0) {
248 printf("%-40s %16llu %12llu %12llu %9.4f %9.2f\n",
250 stat.total_work_size,
251 stat.num_dispatches,
252 stat.total_work_size / stat.num_dispatches,
253 stat.total_time,
254 stat.total_time * 100.0 / total_time);
255 }
256 if (has_extra && i == DEVICE_KERNEL_INTEGRATOR_RESET) {
257 printf("%s\n", divider.c_str());
258 }
259 }
260 printf("%s\n", divider.c_str());
261 printf("%-40s %16s %12llu %12s %9.4f %9.2f\n", "", "", num_dispatches, "", total_time, 100.0);
262 printf("%s\n\n", divider.c_str());
263 }
264}
265
266ConcurrentStatesParams MetalDeviceQueue::concurrent_states_params() const
267{
269 params.baseline = 4194304;
270
271 /* Increasing the state count doesn't notably benefit M1-family systems. */
272 if (MetalInfo::get_apple_gpu_architecture(metal_device_->mtlDevice) == APPLE_M1) {
273 params.min = params.baseline;
274 params.max = params.baseline;
275 return params;
276 }
277
278 /* Keep 10% of the working set free, and only grow within half of the remaining memory.
279 * Limit to 4x the baseline, we see diminishing returns after that. */
280 params.max = params.baseline * 4;
281 return params;
282}
283
284void MetalDeviceQueue::get_memory_info(size_t &total, size_t &free) const
285{
286 total = [metal_device_->mtlDevice recommendedMaxWorkingSetSize];
287 free = total - std::min(stats_.mem_used, total);
288}
289
290int MetalDeviceQueue::num_sort_partitions(int max_num_paths, uint max_scene_shaders) const
291{
292 int sort_partition_elements = MetalInfo::optimal_sort_partition_elements();
293 /* Sort partitioning becomes less effective when more shaders are in the wavefront. In lieu of
294 * a more sophisticated heuristic we simply disable sort partitioning if the shader count is
295 * high.
296 */
297 if (max_scene_shaders < 300 && sort_partition_elements > 0) {
298 return max(max_num_paths / sort_partition_elements, 1);
299 }
300 else {
301 return 1;
302 }
303}
304
305bool MetalDeviceQueue::supports_local_atomic_sort() const
306{
307 return metal_device_->use_local_atomic_sort();
308}
309
310static void zero_resource(void *address_in_arg_buffer, int index = 0)
311{
312 uint64_t *pptr = (uint64_t *)address_in_arg_buffer;
313 pptr[index] = 0;
314}
315
316template<class T> void write_resource(void *address_in_arg_buffer, T resource, int index = 0)
317{
318 zero_resource(address_in_arg_buffer, index);
319 uint64_t *pptr = (uint64_t *)address_in_arg_buffer;
320 if (resource) {
321 MTLResourceID resource_id = resource.gpuResourceID;
322 pptr[index] = (uint64_t &)resource_id;
323 }
324}
325
326template<> void write_resource(void *address_in_arg_buffer, id<MTLBuffer> buffer, int index)
327{
328 zero_resource(address_in_arg_buffer, index);
329 uint64_t *pptr = (uint64_t *)address_in_arg_buffer;
330 if (buffer) {
331 pptr[index] = buffer.gpuAddress;
332 }
333}
334
335static id<MTLBuffer> patch_resource(void *address_in_arg_buffer, int index = 0)
336{
337 uint64_t *pptr = (uint64_t *)address_in_arg_buffer;
338 if (MetalDevice::MetalMem *mmem = (MetalDevice::MetalMem *)pptr[index]) {
339 write_resource<id<MTLBuffer>>(address_in_arg_buffer, mmem->mtlBuffer, index);
340 return mmem->mtlBuffer;
341 }
342 return nil;
343}
344
345void MetalDeviceQueue::init_execution()
346{
347 /* Populate blas_array. */
348 uint64_t *blas_array = (uint64_t *)metal_device_->blas_buffer.contents;
349 for (uint64_t slot = 0; slot < metal_device_->blas_array.size(); ++slot) {
350 write_resource(blas_array, metal_device_->blas_array[slot], slot);
351 }
352
353 /* Populate image bindings. */
354 load_image_info();
355
356 /* Synchronize memory copies. */
357 synchronize();
358}
359
360void MetalDeviceQueue::load_image_info()
361{
362 /* TODO: Can this be optimized to only update info ids that changed? */
363
364 /* Buffers below are modified and possibly reallocated, so pending kernels must finish first.
365 * This is why it's done here rather than in image_alloc(). */
366 synchronize();
367
368 /* CPU render threads may allocate images on this device concurrently, which can grow
369 * image_info and image_info_id_map on the host. */
370 std::lock_guard<std::recursive_mutex> lock(metal_device_->metal_mem_map_mutex);
371
372 device_vector<KernelImageInfo> &image_info = metal_device_->image_info;
373 id<MTLBuffer> &image_bindings = metal_device_->image_bindings;
374 std::vector<id<MTLResource>> &image_info_id_map = metal_device_->image_info_id_map;
375
376 /* Reallocate device buffer if image_info was grown with host_only_resize(). */
377 if (image_info.device_pointer && image_info.host_pointer != image_info.shared_pointer) {
378 metal_device_->generic_free(image_info);
379 }
380
381 const size_t min_bindings_length = sizeof(void *) * image_info.size();
382 if (image_bindings.length < min_bindings_length) {
383 metal_device_->metal_mem_free(image_bindings);
384 image_bindings = [metal_device_->mtlDevice newBufferWithLength:min_bindings_length
385 options:MTLResourceStorageModeShared];
386 metal_device_->metal_mem_alloc(image_bindings);
387 }
388
389 /* Ensure image_info is allocated before populating. */
390 image_info.copy_to_device();
391
392 /* Populate texture bindings. */
393 uint64_t *bindings = (uint64_t *)image_bindings.contents;
394 memset(bindings, 0, image_bindings.length);
395 for (int image_info_id = 0; image_info_id < image_info.size(); ++image_info_id) {
396 if (image_info_id_map[image_info_id]) {
397 if (metal_device_->is_texture(image_info[image_info_id])) {
398 write_resource(bindings, id<MTLTexture>(image_info_id_map[image_info_id]), image_info_id);
399 }
400 else {
401 /* The GPU address of a 1D buffer texture is written into the image_info_id data field. */
402 write_resource(
403 &image_info[image_info_id].data, id<MTLBuffer>(image_info_id_map[image_info_id]), 0);
404 }
405 }
406 }
407}
408
409bool MetalDeviceQueue::enqueue(DeviceKernel kernel,
410 const int work_size,
411 const DeviceKernelArguments &args)
412{
413 @autoreleasepool {
414 update_capture(kernel);
415
416 if (metal_device_->have_error()) {
417 return false;
418 }
419
420 debug_enqueue_begin(kernel, work_size);
421
422 LOG_TRACE << "Metal queue launch " << device_kernel_as_string(kernel) << ", work_size "
423 << work_size;
424
425 id<MTLComputeCommandEncoder> mtlComputeCommandEncoder = get_compute_encoder(kernel);
426
427 if (profiling_enabled_) {
428 command_encoder_labels_.push_back({kernel, work_size, current_encoder_idx_});
429 }
430 if (label_command_encoders_) {
431 /* Add human-readable labels if we're doing any form of debugging / profiling. */
432 mtlComputeCommandEncoder.label = [NSString
433 stringWithFormat:@"Metal queue launch %s, work_size %d",
435 work_size];
436 }
437
438 if (!active_pipelines_[kernel].update(metal_device_, kernel)) {
439 metal_device_->set_error(
440 string_printf("Could not activate pipeline for %s\n", device_kernel_as_string(kernel)));
441 return false;
442 }
443 MetalDispatchPipeline &active_pipeline = active_pipelines_[kernel];
444
445 uint8_t dynamic_args[512] = {0};
446
447 /* Prepare the dynamic "enqueue" arguments */
448 size_t dynamic_bytes_written = 0;
449 size_t max_size_in_bytes = 0;
450 for (size_t i = 0; i < args.count; i++) {
451 size_t size_in_bytes = args.sizes[i];
452 max_size_in_bytes = max(max_size_in_bytes, size_in_bytes);
453 dynamic_bytes_written = round_up(dynamic_bytes_written, size_in_bytes);
454 memcpy(dynamic_args + dynamic_bytes_written, args.values[i], size_in_bytes);
455 if (args.types[i] == DeviceKernelArguments::POINTER) {
456 patch_resource(dynamic_args + dynamic_bytes_written);
457 }
458 dynamic_bytes_written += size_in_bytes;
459 }
460 /* Apply conventional struct alignment (stops asserts firing when API validation is enabled).
461 */
462 dynamic_bytes_written = round_up(dynamic_bytes_written, max_size_in_bytes);
463
464 /* Check that the dynamic args didn't overflow. */
465 assert(dynamic_bytes_written <= sizeof(dynamic_args));
466
467 uint64_t ancillary_args[ANCILLARY_SLOT_COUNT] = {0};
468
469 /* Encode ancillaries */
470 int ancillary_index = 0;
471 write_resource(ancillary_args, metal_device_->image_bindings, ancillary_index++);
472
473 if (metal_device_->use_metalrt) {
474 write_resource(ancillary_args, metal_device_->accel_struct, ancillary_index++);
475 write_resource(ancillary_args, metal_device_->blas_buffer, ancillary_index++);
476
477 /* Write the intersection function table. */
478 for (int table_idx = 0; table_idx < METALRT_TABLE_NUM; table_idx++) {
479 write_resource(
480 ancillary_args, active_pipeline.intersection_func_table[table_idx], ancillary_index++);
481 }
482 assert(ancillary_index == ANCILLARY_SLOT_COUNT);
483 }
484
485 [mtlComputeCommandEncoder setBytes:dynamic_args length:dynamic_bytes_written atIndex:0];
486 [mtlComputeCommandEncoder setBuffer:metal_device_->launch_params_buffer offset:0 atIndex:1];
487 [mtlComputeCommandEncoder setBytes:ancillary_args length:sizeof(ancillary_args) atIndex:2];
488
489 /* Fallback path in case residency sets aren't supported:
490 * Call useResource for MetalRT resources not covered by prepare_resources(). */
491 if (!metal_device_->mtlResidencySet_enabled && metal_device_->use_metalrt &&
493 {
494 if (id<MTLAccelerationStructure> accel_struct = metal_device_->accel_struct) {
495 /* Mark all Accelerations resources as used */
496 [mtlComputeCommandEncoder useResource:accel_struct usage:MTLResourceUsageRead];
497 if (metal_device_->blas_buffer) {
498 [mtlComputeCommandEncoder useResource:metal_device_->blas_buffer
499 usage:MTLResourceUsageRead];
500 }
501 [mtlComputeCommandEncoder useResources:metal_device_->unique_blas_array.data()
502 count:metal_device_->unique_blas_array.size()
503 usage:MTLResourceUsageRead];
504 }
505
506 for (int table = 0; table < METALRT_TABLE_NUM; table++) {
507 if (active_pipeline.intersection_func_table[table]) {
508 [mtlComputeCommandEncoder useResource:active_pipeline.intersection_func_table[table]
509 usage:MTLResourceUsageRead];
510 }
511 }
512 }
513
514 [mtlComputeCommandEncoder setComputePipelineState:active_pipeline.pipeline];
515
516 /* Compute kernel launch parameters. */
517 const int num_threads_per_block = active_pipeline.num_threads_per_block;
518
519 int shared_mem_bytes = 0;
520
521 switch (kernel) {
530 /* See parallel_active_index.h for why this amount of shared memory is needed.
531 * Rounded up to 16 bytes for Metal */
532 shared_mem_bytes = (int)round_up((num_threads_per_block + 1) * sizeof(int), 16);
533 break;
534
537 int key_count = metal_device_->launch_params->data.max_shaders;
538 shared_mem_bytes = (int)round_up(key_count * sizeof(int), 16);
539 break;
540 }
541
542 default:
543 break;
544 }
545
546 if (shared_mem_bytes) {
547 assert(shared_mem_bytes <= 32 * 1024);
548 [mtlComputeCommandEncoder setThreadgroupMemoryLength:shared_mem_bytes atIndex:0];
549 }
550
551 MTLSize size_threads_per_dispatch = MTLSizeMake(work_size, 1, 1);
552 MTLSize size_threads_per_threadgroup = MTLSizeMake(num_threads_per_block, 1, 1);
553 [mtlComputeCommandEncoder dispatchThreads:size_threads_per_dispatch
554 threadsPerThreadgroup:size_threads_per_threadgroup];
555
556 metal_device_->prepare_residency();
557
558 [mtlCommandBuffer_ addCompletedHandler:^(id<MTLCommandBuffer> command_buffer) {
559 /* Enhanced command buffer errors */
560 string str;
561 if (command_buffer.status != MTLCommandBufferStatusCompleted) {
562 str = string_printf("Command buffer not completed. status = %d. ",
563 int(command_buffer.status));
564 }
565 if (command_buffer.error) {
566 @autoreleasepool {
567 const char *errCStr = [[NSString stringWithFormat:@"%@", command_buffer.error]
568 UTF8String];
569 str += string_printf("(%s.%s):\n%s\n",
570 kernel_type_as_string(active_pipeline.pso_type),
572 errCStr);
573 }
574 }
575 if (!str.empty()) {
576 metal_device_->set_error(str);
577 }
578 }];
579
580 if (verbose_tracing_ || is_capturing_) {
581 /* Force a sync we've enabled step-by-step verbose tracing or if we're capturing. */
582 synchronize();
583
584 /* Show queue counters and dispatch timing. */
585 if (verbose_tracing_) {
586 if (kernel == DEVICE_KERNEL_INTEGRATOR_RESET) {
587 printf(
588 "_____________________________________.____________________.______________._________"
589 "__"
590 "______________________________________\n");
591 }
592
593 printf("%-40s| %7d threads |%5.2fms | buckets [",
595 work_size,
596 last_completion_time_ * 1000.0);
597 std::lock_guard<std::recursive_mutex> lock(metal_device_->metal_mem_map_mutex);
598 for (auto &it : metal_device_->metal_mem_map) {
599 const string c_integrator_queue_counter = "integrator_queue_counter";
600 if (it.first->global_name() == c_integrator_queue_counter) {
601 if (IntegratorQueueCounter *queue_counter = (IntegratorQueueCounter *)
602 it.first->host_pointer)
603 {
604 for (int i = 0; i < DEVICE_GPU_KERNEL_INTEGRATOR_NUM; i++) {
605 printf("%s%d", i == 0 ? "" : ",", queue_counter->num_queued[i]);
606 }
607 }
608 break;
609 }
610 }
611 printf("]\n");
612 }
613 }
614
615 debug_enqueue_end();
616
617 return !(metal_device_->have_error());
618 }
619}
620
621void MetalDeviceQueue::flush_timing_stats()
622{
623 for (auto label : command_encoder_labels_) {
624 TimingStats &stat = timing_stats_[label.kernel];
625
626 double completion_time_gpu;
627 NSData *computeTimeStamps = [metal_device_->mtlCounterSampleBuffer
628 resolveCounterRange:NSMakeRange(label.timing_id, 2)];
629 MTLCounterResultTimestamp *timestamps = (MTLCounterResultTimestamp *)(computeTimeStamps.bytes);
630
631 uint64_t begTime = timestamps[0].timestamp;
632 uint64_t endTime = timestamps[1].timestamp;
633 completion_time_gpu = (endTime - begTime) / (double)NSEC_PER_SEC;
634
635 stat.num_dispatches++;
636 stat.total_time += completion_time_gpu;
637 stat.total_work_size += label.work_size;
638 last_completion_time_ = completion_time_gpu;
639 }
640 command_encoder_labels_.clear();
641}
642
643bool MetalDeviceQueue::synchronize()
644{
645 @autoreleasepool {
646 if (has_captured_to_disk_ || metal_device_->have_error()) {
647 return false;
648 }
649
650 close_compute_encoder();
651 close_blit_encoder();
652
653 if (mtlCommandBuffer_) {
654 scoped_timer timer;
655
656 uint64_t shared_event_id_ = this->shared_event_id_++;
657
658 __block dispatch_semaphore_t block_sema = wait_semaphore_;
659 [shared_event_ notifyListener:shared_event_listener_
660 atValue:shared_event_id_
661 block:^(id<MTLSharedEvent> /*sharedEvent*/, uint64_t /*value*/) {
662 dispatch_semaphore_signal(block_sema);
663 }];
664
665 [mtlCommandBuffer_ encodeSignalEvent:shared_event_ value:shared_event_id_];
666 [mtlCommandBuffer_ commit];
667 dispatch_semaphore_wait(wait_semaphore_, DISPATCH_TIME_FOREVER);
668
669 [mtlCommandBuffer_ release];
670
671 metal_device_->flush_delayed_free_list();
672
673 mtlCommandBuffer_ = nil;
674 flush_timing_stats();
675 }
676
677 debug_synchronize();
678
679 return !(metal_device_->have_error());
680 }
681}
682
683void MetalDeviceQueue::zero_to_device(device_memory &mem)
684{
685 @autoreleasepool {
686 if (metal_device_->have_error()) {
687 return;
688 }
689
691
692 if (mem.memory_size() == 0) {
693 return;
694 }
695
696 /* Allocate on demand. */
697 if (mem.device_pointer == 0) {
698 metal_device_->mem_alloc(mem);
699 }
700
701 /* Zero memory on device. */
702 assert(mem.device_pointer != 0);
703
704 std::lock_guard<std::recursive_mutex> lock(metal_device_->metal_mem_map_mutex);
705 MetalDevice::MetalMem &mmem = *metal_device_->metal_mem_map.at(&mem);
706 if (mmem.mtlBuffer) {
707 id<MTLBlitCommandEncoder> blitEncoder = get_blit_encoder();
708 [blitEncoder fillBuffer:mmem.mtlBuffer range:NSMakeRange(mmem.offset, mmem.size) value:0];
709 }
710 else {
711 metal_device_->mem_zero(mem);
712 }
713 }
714}
715
716void MetalDeviceQueue::copy_to_device(device_memory &mem)
717{
718 @autoreleasepool {
719 if (metal_device_->have_error()) {
720 return;
721 }
722
723 if (mem.memory_size() == 0) {
724 return;
725 }
726
727 /* Allocate on demand. */
728 if (mem.device_pointer == 0) {
729 metal_device_->mem_alloc(mem);
730 }
731
732 assert(mem.device->mem_device_ptr(mem, metal_device_) != 0);
733 assert(mem.host_pointer != nullptr);
734 /* No need to copy - Apple Silicon has Unified Memory Architecture. */
735 }
736}
737
738void MetalDeviceQueue::copy_from_device(device_memory & /*mem*/)
739{
740 /* No need to copy - Apple Silicon has Unified Memory Architecture. */
741}
742
743void *MetalDeviceQueue::copy_from_device_synchronized(device_memory &mem,
744 vector<uint8_t> & /*storage*/)
745{
746 if (mem.memory_size() == 0) {
747 return nullptr;
748 }
749
750 /* Wait until kernels have finished before returning from unified memory. */
751 synchronize();
752
753 device_ptr d_ptr = mem.device->mem_device_ptr(mem, metal_device_);
754 return (d_ptr) ? reinterpret_cast<MetalDevice::MetalMem *>(d_ptr)->hostPtr : nullptr;
755}
756
757void MetalDeviceQueue::prepare_resources()
758{
759 if (metal_device_->mtlResidencySet_enabled) {
760 /* All resources are already resident - skip per-encoder useResource calls. */
761 return;
762 }
763
764 std::lock_guard<std::recursive_mutex> lock(metal_device_->metal_mem_map_mutex);
765
766 /* declare resource usage */
767 for (auto &it : metal_device_->metal_mem_map) {
768 device_memory *mem = it.first;
769
770 MTLResourceUsage usage = MTLResourceUsageRead;
771 if (mem->type != MEM_GLOBAL && mem->type != MEM_READ_ONLY && mem->type != MEM_IMAGE_TEXTURE) {
772 usage |= MTLResourceUsageWrite;
773 }
774
775 if (it.second->mtlBuffer) {
776 /* METAL_WIP - use array version (i.e. useResources) */
777 [mtlComputeEncoder_ useResource:it.second->mtlBuffer usage:usage];
778 }
779 else if (it.second->mtlTexture) {
780 /* METAL_WIP - use array version (i.e. useResources) */
781 [mtlComputeEncoder_ useResource:it.second->mtlTexture usage:usage];
782 }
783 }
784
785 /* ancillaries */
786 [mtlComputeEncoder_ useResource:metal_device_->image_bindings usage:MTLResourceUsageRead];
787}
788
789id<MTLComputeCommandEncoder> MetalDeviceQueue::get_compute_encoder(DeviceKernel kernel)
790{
791 bool concurrent = int(kernel) < int(DEVICE_GPU_KERNEL_INTEGRATOR_NUM);
792
793 if (profiling_enabled_) {
794 /* Close the current encoder to ensure we're able to capture per-encoder timing data. */
795 close_compute_encoder();
796 }
797
798 if (mtlComputeEncoder_) {
799 if (mtlComputeEncoder_.dispatchType == concurrent ? MTLDispatchTypeConcurrent :
800 MTLDispatchTypeSerial)
801 {
802 /* declare usage of MTLBuffers etc */
803 prepare_resources();
804
805 return mtlComputeEncoder_;
806 }
807 close_compute_encoder();
808 }
809
810 close_blit_encoder();
811
812 if (!mtlCommandBuffer_) {
813 mtlCommandBuffer_ = [mtlCommandQueue_ commandBuffer];
814 [mtlCommandBuffer_ retain];
815 }
816
817 if (profiling_enabled_) {
818 MTLComputePassDescriptor *desc = [[MTLComputePassDescriptor alloc] init];
819
820 current_encoder_idx_ = (counter_sample_buffer_curr_idx_.fetch_add(2) %
821 MAX_SAMPLE_BUFFER_LENGTH);
822 [desc.sampleBufferAttachments[0] setSampleBuffer:metal_device_->mtlCounterSampleBuffer];
823 [desc.sampleBufferAttachments[0] setStartOfEncoderSampleIndex:current_encoder_idx_];
824 [desc.sampleBufferAttachments[0] setEndOfEncoderSampleIndex:current_encoder_idx_ + 1];
825
826 [desc setDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
827
828 mtlComputeEncoder_ = [mtlCommandBuffer_ computeCommandEncoderWithDescriptor:desc];
829 }
830 else {
831 mtlComputeEncoder_ = [mtlCommandBuffer_
832 computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent :
833 MTLDispatchTypeSerial];
834 }
835
836 [mtlComputeEncoder_ retain];
837 [mtlComputeEncoder_ setLabel:@(device_kernel_as_string(kernel))];
838
839 /* declare usage of MTLBuffers etc */
840 prepare_resources();
841
842 return mtlComputeEncoder_;
843}
844
845id<MTLBlitCommandEncoder> MetalDeviceQueue::get_blit_encoder()
846{
847 if (mtlBlitEncoder_) {
848 return mtlBlitEncoder_;
849 }
850
851 close_compute_encoder();
852
853 if (!mtlCommandBuffer_) {
854 mtlCommandBuffer_ = [mtlCommandQueue_ commandBuffer];
855 [mtlCommandBuffer_ retain];
856 }
857
858 mtlBlitEncoder_ = [mtlCommandBuffer_ blitCommandEncoder];
859 [mtlBlitEncoder_ retain];
860 return mtlBlitEncoder_;
861}
862
863void MetalDeviceQueue::close_compute_encoder()
864{
865 if (mtlComputeEncoder_) {
866 [mtlComputeEncoder_ endEncoding];
867 [mtlComputeEncoder_ release];
868 mtlComputeEncoder_ = nil;
869 }
870}
871
872void MetalDeviceQueue::close_blit_encoder()
873{
874 if (mtlBlitEncoder_) {
875 [mtlBlitEncoder_ endEncoding];
876 [mtlBlitEncoder_ release];
877 mtlBlitEncoder_ = nil;
878 }
879}
880
881void *MetalDeviceQueue::native_queue()
882{
883 return mtlCommandQueue_;
884}
885
886unique_ptr<DeviceGraphicsInterop> MetalDeviceQueue::graphics_interop_create()
887{
888 return make_unique<MetalDeviceGraphicsInterop>(this);
889}
890
892
893#endif /* WITH_METAL */
volatile int lock
void init()
long long int int64_t
unsigned long long int uint64_t
btScalar max
virtual device_ptr mem_device_ptr(const device_memory &mem, Device *sub_device)
size_t size() const
#define printf
@ MEM_IMAGE_TEXTURE
@ MEM_READ_ONLY
CCL_NAMESPACE_BEGIN struct Options options
#define CCL_NAMESPACE_END
bool device_kernel_has_intersection(DeviceKernel kernel)
const char * device_kernel_as_string(DeviceKernel kernel)
#define str(s)
#define assert(assertion)
#define INT_MAX
#define index(i)
float length(VecOp< float, D >) RET
ccl_gpu_kernel_postfix const ccl_global int ccl_global float const int work_size
@ DEVICE_GPU_KERNEL_INTEGRATOR_NUM
DeviceKernel
@ DEVICE_KERNEL_INTEGRATOR_RESET
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORT_WRITE_PASS
@ DEVICE_KERNEL_INTEGRATOR_QUEUED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORTED_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_SORT_BUCKET_PASS
@ DEVICE_KERNEL_INTEGRATOR_TERMINATED_SHADOW_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL
@ DEVICE_KERNEL_NUM
@ DEVICE_KERNEL_INTEGRATOR_ACTIVE_PATHS_ARRAY
@ DEVICE_KERNEL_INTEGRATOR_COMPACT_PATHS_ARRAY
#define LOG_TRACE
Definition log.h:108
#define T
void free(struct bMotionPath *mpath)
unsigned char uint8_t
Definition BKE_text.h:79
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
Definition string.cpp:25
void * values[MAX_ARGS]
size_t sizes[MAX_ARGS]
Type types[MAX_ARGS]
ccl_device_inline size_t round_up(const size_t x, const size_t multiple)
Definition types_base.h:66
uint64_t device_ptr
Definition types_base.h:44