Blender V5.3
mtl_memory.mm
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2022-2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include "BKE_global.hh"
10
11#include "DNA_userdef_types.h"
12
13#include "BLI_math_base_c.hh"
14
15#include "mtl_context.hh"
16#include "mtl_debug.hh"
17#include "mtl_memory.hh"
18#include "mtl_storage_buffer.hh"
19
20using namespace blender::gpu;
21
22/* Allows a scratch buffer to temporarily grow beyond its maximum, which allows submission
23 * of one-time-use data packets which are too large. */
24#define MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION
25
26/* Memory size in bytes macros, used as pool flushing frequency thresholds. */
27constexpr static size_t MEMORY_SIZE_256MB = 256LL * (1024LL * 1024LL);
28constexpr static size_t MEMORY_SIZE_512MB = 512LL * (1024LL * 1024LL);
29constexpr static size_t MEMORY_SIZE_1GB = 1LL * (1024LL * 1024LL * 1024LL);
30constexpr static size_t MEMORY_SIZE_2GB = 2LL * (1024LL * 1024LL * 1024LL);
31
32namespace blender::gpu {
33
34static CLG_LogRef LOG = {"gpu.metal"};
35
36/* -------------------------------------------------------------------- */
39
41{
42 if (!initialized_) {
43 BLI_assert(mtl_device);
44 initialized_ = true;
45 device_ = mtl_device;
46
47#if MTL_DEBUG_MEMORY_STATISTICS == 1
48 /* Debug statistics. */
49 total_allocation_bytes_ = 0;
50 per_frame_allocation_count_ = 0;
51 buffers_in_pool_ = 0;
52#endif
53 /* Track pool allocation size. */
54 allocations_in_pool_ = 0;
55
56 /* Live allocations list. */
57 allocations_list_base_ = nullptr;
58 allocations_list_size_ = 0;
59
60 /* Free pools -- Create initial safe free pool */
61 BLI_assert(current_free_list_ == nullptr);
62 this->begin_new_safe_list();
63 }
64}
65
67{
68 this->free();
69}
70
71void MTLBufferPool::free()
72{
73 buffer_pool_lock_.lock();
74
75 /* Delete all existing allocations. */
76 allocations_list_delete_all();
77
78 /* Release safe free lists. */
79 for (int safe_pool_free_index = 0; safe_pool_free_index < completed_safelist_queue_.size();
80 safe_pool_free_index++)
81 {
82 delete completed_safelist_queue_[safe_pool_free_index];
83 }
84 completed_safelist_queue_.clear();
85
86 safelist_lock_.lock();
87 if (current_free_list_ != nullptr) {
88 delete current_free_list_;
89 current_free_list_ = nullptr;
90 }
91 if (prev_free_buffer_list_ != nullptr) {
92 delete prev_free_buffer_list_;
93 prev_free_buffer_list_ = nullptr;
94 }
95 safelist_lock_.unlock();
96
97 /* Clear and release memory pools. */
98 for (std::multiset<blender::gpu::MTLBufferHandle, blender::gpu::CompareMTLBuffer> *buffer_pool :
99 buffer_pools_.values())
100 {
101 delete buffer_pool;
102 }
103
104 buffer_pools_.clear();
105 buffer_pool_lock_.unlock();
106}
107
109{
110 /* Allocate buffer with default HW-compatible alignment of 256 bytes.
111 * See https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf for more. */
112 return this->allocate_aligned(size, 256, cpu_visible);
113}
114
116 bool cpu_visible,
117 const void *data)
118{
119 /* Allocate buffer with default HW-compatible alignment of 256 bytes.
120 * See https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf for more. */
121 return this->allocate_aligned_with_data(size, 256, cpu_visible, data);
122}
123
125 uint32_t alignment,
126 bool cpu_visible)
127{
128 /* Check not required. Main GPU module usage considered thread-safe. */
129 // BLI_assert(BLI_thread_is_main());
130
131 /* Calculate aligned size */
132 BLI_assert(alignment > 0);
133 uint64_t aligned_alloc_size = ceil_to_multiple_ul(size, alignment);
134
135 /* Allocate new MTL Buffer */
136 MTLResourceOptions options;
137 if (cpu_visible) {
138 options = ([device_ hasUnifiedMemory]) ? MTLResourceStorageModeShared :
139 MTLResourceStorageModeManaged;
140 }
141 else {
142 options = MTLResourceStorageModePrivate;
143 }
144
145 /* Check if we have a suitable buffer */
146 gpu::MTLBuffer *new_buffer = nullptr;
147 buffer_pool_lock_.lock();
148
149 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.lookup_ptr(
151
152 if (pool_search != nullptr) {
153 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool = *pool_search;
154 MTLBufferHandle size_compare(aligned_alloc_size);
155 auto result = pool->lower_bound(size_compare);
156 if (result != pool->end()) {
157 /* Potential buffer found, check if within size threshold requirements. */
158 gpu::MTLBuffer *found_buffer = result->buffer;
159 BLI_assert(found_buffer);
160 BLI_assert(found_buffer->get_metal_buffer());
161
162 uint64_t found_size = found_buffer->get_size();
163
164 if (found_size >= aligned_alloc_size &&
165 found_size <= (aligned_alloc_size * mtl_buffer_size_threshold_factor_))
166 {
168 "[MemoryAllocator] Suitable Buffer of size %lld found, for requested size: %lld",
169 found_size,
170 aligned_alloc_size);
171
172 new_buffer = found_buffer;
173 BLI_assert(!new_buffer->get_in_use());
174
175 /* Remove buffer from free set. */
176 pool->erase(result);
177 }
178 else {
180 "[MemoryAllocator] Buffer of size %lld found, but was incompatible with requested "
181 "size: %lld",
182 found_size,
183 aligned_alloc_size);
184 new_buffer = nullptr;
185 }
186 }
187 }
188
189 /* Allocate new buffer. */
190 if (new_buffer == nullptr) {
191 new_buffer = new gpu::MTLBuffer(device_, size, options, alignment);
192
193 /* Track allocation in context. */
194 allocations_list_insert(new_buffer);
195 }
196 else {
197 /* Re-use suitable buffer. */
198 new_buffer->set_usage_size(aligned_alloc_size);
199
200#if MTL_DEBUG_MEMORY_STATISTICS == 1
201 /* Debug. */
202 buffers_in_pool_--;
203#endif
204
205 /* Decrement size of pool. */
206 BLI_assert(allocations_in_pool_ >= 0);
207 allocations_in_pool_ -= new_buffer->get_size();
208
209 /* Ensure buffer memory is correctly backed. */
210 BLI_assert(new_buffer->get_metal_buffer());
211 }
212 /* Flag buffer as actively in-use. */
213 new_buffer->flag_in_use(true);
214
215#if MTL_DEBUG_MEMORY_STATISTICS == 1
216 per_frame_allocation_count_++;
217#endif
218
219 /* Release lock. */
220 buffer_pool_lock_.unlock();
221
222 return new_buffer;
223}
224
226 uint32_t /*alignment*/,
227 bool cpu_visible,
228 const void *data)
229{
230 gpu::MTLBuffer *buf = this->allocate_aligned(size, 256, cpu_visible);
231
232 /* Upload initial data. */
233 BLI_assert(data != nullptr);
234 BLI_assert(!(buf->get_resource_options() & MTLResourceStorageModePrivate));
235 BLI_assert(size <= buf->get_size());
236 BLI_assert(size <= [buf->get_metal_buffer() length]);
237 memcpy(buf->get_host_ptr(), data, size);
238 buf->flush_range(0, size);
239 return buf;
240}
241
243{
244 /* Ensure buffer is flagged as in-use. I.e. has not already been returned to memory pools. */
245 bool buffer_in_use = buffer->get_in_use();
246 BLI_assert(buffer_in_use);
247 if (buffer_in_use) {
248
249 /* Reference the active safe pool while the buffer is inserted, otherwise other threads can
250 * begin new lists and release it in the meantime. Locked so that it can't be replaced and
251 * released before it is referenced. This doesn't flag the list as referenced by a workload,
252 * see #MTLSafeFreeList::should_flush. */
253 safelist_lock_.lock();
254 MTLSafeFreeList *current_pool = this->get_current_safe_list();
255 BLI_assert(current_pool);
256 current_pool->reference_count_++;
257 safelist_lock_.unlock();
258
259 /* Place buffer in safe_free_pool before returning to MemoryManager buffer pools. */
260 current_pool->insert_buffer(buffer);
261 buffer->flag_in_use(false);
262
263 /* Release after the buffer is flagged as unused, the list can be completed here. */
264 current_pool->decrement_reference();
265
266 return true;
267 }
268 return false;
269}
270
272{
273 /* Ensure thread-safe access to `completed_safelist_queue_`, which contains
274 * the list of MTLSafeFreeList's whose buffers are ready to be
275 * re-inserted into the Memory Manager pools.
276 * we also need to lock access to general buffer pools, to ensure allocations
277 * are not simultaneously happening on background threads. */
278 safelist_lock_.lock();
279 buffer_pool_lock_.lock();
280
281#if MTL_DEBUG_MEMORY_STATISTICS == 1
282 int num_buffers_added = 0;
283#endif
284
285 /* Always free oldest MTLSafeFreeList first. */
286 for (int safe_pool_free_index = 0; safe_pool_free_index < completed_safelist_queue_.size();
287 safe_pool_free_index++)
288 {
289 MTLSafeFreeList *current_pool = completed_safelist_queue_[safe_pool_free_index];
290
291 /* Iterate through all MTLSafeFreeList linked-chunks. */
292 while (current_pool != nullptr) {
293 current_pool->lock_.lock();
294 BLI_assert(current_pool);
295 BLI_assert(current_pool->in_free_queue_);
296 int counter = 0;
297 int size = min_ii(current_pool->current_list_index_, MTLSafeFreeList::MAX_NUM_BUFFERS_);
298
299 /* Re-add all buffers within frame index to MemoryManager pools. */
300 while (counter < size) {
301
302 gpu::MTLBuffer *buf = current_pool->safe_free_pool_[counter];
303
304 /* Insert buffer back into open pools. */
305 BLI_assert(buf->get_in_use() == false);
306 this->insert_buffer_into_pool(buf->get_resource_options(), buf);
307 counter++;
308
309#if MTL_DEBUG_MEMORY_STATISTICS == 1
310 num_buffers_added++;
311#endif
312 }
313
314 /* Fetch next MTLSafeFreeList chunk, if any. */
315 MTLSafeFreeList *next_list = current_pool->next_.load();
316
317 /* Delete current MTLSafeFreeList */
318 current_pool->lock_.unlock();
319 delete current_pool;
320 current_pool = nullptr;
321
322 /* Move onto next chunk. */
323 if (next_list != nullptr) {
324 current_pool = next_list;
325 }
326 }
327 }
328
329 /* Release memory allocations which have not been used in a while.
330 * This ensures memory pressure stays low for scenes with compounding complexity during
331 * animation.
332 * If memory is continually used, then we do not want to free this memory as it will be
333 * re-allocated during a short time period. */
334
335 const time_t time_now = std::time(nullptr);
336 for (auto buffer_pool_list : buffer_pools_.items()) {
337 MTLBufferPoolOrderedList *pool_allocations = buffer_pool_list.value;
338 MTLBufferPoolOrderedList::iterator pool_iterator = pool_allocations->begin();
339 while (pool_iterator != pool_allocations->end()) {
340
341 const MTLBufferHandle handle = *pool_iterator;
342 const time_t time_passed = time_now - handle.insert_time;
343
344 /* Free allocations if a certain amount of time has passed.
345 * Deletion frequency depends on how much excess memory
346 * the application is using. */
347 time_t deletion_time_threshold_s = 600;
348 /* Spare pool memory >= 2GB. */
349 if (allocations_in_pool_ >= MEMORY_SIZE_2GB) {
350 deletion_time_threshold_s = 2;
351 }
352 else
353 /* Spare pool memory >= 1GB. */
354 if (allocations_in_pool_ >= MEMORY_SIZE_1GB) {
355 deletion_time_threshold_s = 4;
356 }
357 /* Spare pool memory >= 512MB. */
358 else if (allocations_in_pool_ >= MEMORY_SIZE_512MB) {
359 deletion_time_threshold_s = 15;
360 }
361 /* Spare pool memory >= 256MB. */
362 else if (allocations_in_pool_ >= MEMORY_SIZE_256MB) {
363 deletion_time_threshold_s = 60;
364 }
365
366 if (time_passed > deletion_time_threshold_s) {
367
368 /* Remove buffer from global allocations list and release resource. */
369 allocations_list_delete(handle.buffer);
370
371 /* Remove buffer from pool and update pool statistics. */
372 pool_iterator = pool_allocations->erase(pool_iterator);
373 allocations_in_pool_ -= handle.buffer_size;
374#if MTL_DEBUG_MEMORY_STATISTICS == 1
375 buffers_in_pool_--;
376#endif
377 continue;
378 }
379 pool_iterator++;
380 }
381 }
382
383#if MTL_DEBUG_MEMORY_STATISTICS == 1
384 CLOG_DEBUG(&LOG, "--- Allocation Stats ---");
385 CLOG_DEBUG(&LOG, " Num buffers processed in pool (this frame): %u", num_buffers_added);
386
387 uint framealloc = (uint)per_frame_allocation_count_;
388 CLOG_DEBUG(&LOG, " Allocations in frame: %u", framealloc);
389 CLOG_DEBUG(&LOG, " Total Buffers allocated: %u", allocations_list_size_);
391 &LOG, " Total Memory allocated: %u MB", (uint)total_allocation_bytes_ / (1024 * 1024));
392
393 uint allocs = (uint)(allocations_in_pool_) / 1024 / 2024;
394 CLOG_DEBUG(&LOG, " Free memory in pools: %u MB", allocs);
395
396 uint buffs = (uint)buffers_in_pool_;
397 CLOG_DEBUG(&LOG, " Buffers in pools: %u", buffs);
398
399 CLOG_DEBUG(&LOG, " Pools %u:", (uint)buffer_pools_.size());
400 auto key_iterator = buffer_pools_.keys().begin();
401 auto value_iterator = buffer_pools_.values().begin();
402 while (key_iterator != buffer_pools_.keys().end()) {
403 uint64_t mem_in_pool = 0;
404 uint64_t iters = 0;
405 for (auto it = (*value_iterator)->begin(); it != (*value_iterator)->end(); it++) {
406 mem_in_pool += it->buffer_size;
407 iters++;
408 }
409
411 " Buffers in pool (%u)(%llu): %u (%u MB)",
412 (uint)*key_iterator,
413 iters,
414 (uint)((*value_iterator)->size()),
415 (uint)mem_in_pool / 1024 / 1024);
416 ++key_iterator;
417 ++value_iterator;
418 }
419
420 per_frame_allocation_count_ = 0;
421#endif
422
423 /* Clear safe pools list */
424 completed_safelist_queue_.clear();
425 buffer_pool_lock_.unlock();
426 safelist_lock_.unlock();
427}
428
430{
431 /* When an MTLSafeFreeList has been released by the GPU, and buffers are ready to
432 * be re-inserted into the MemoryManager pools for future use, add the MTLSafeFreeList
433 * to the `completed_safelist_queue_` for flushing at a controlled point in time. */
434 safe_list->lock_.lock();
435 BLI_assert(safe_list);
436 BLI_assert(safe_list->reference_count_ == 0 &&
437 "Pool must be fully dereferenced by all in-use cmd buffers before returning.\n");
438 BLI_assert(safe_list->in_free_queue_ == false && "Pool must not already be in queue");
439
440 /* Flag MTLSafeFreeList as having been added, and insert into SafeFreePool queue. */
441 safe_list->flag_in_queue();
442 safelist_lock_.lock();
443 completed_safelist_queue_.append(safe_list);
444 safelist_lock_.unlock();
445 safe_list->lock_.unlock();
446}
447
449{
450 /* Thread-safe access via atomic ptr. */
451 return current_free_list_;
452}
453
455{
456 /* Locked so that the list can't be replaced and released before it is referenced. */
457 std::scoped_lock lock(safelist_lock_);
459 list->increment_reference();
460 return list;
461}
462
464{
465 std::scoped_lock lock(safelist_lock_);
466 return !completed_safelist_queue_.is_empty();
467}
468
469void MTLBufferPool::begin_new_safe_list(const bool only_if_should_flush)
470{
471 safelist_lock_.lock();
472 MTLSafeFreeList *previous_list = prev_free_buffer_list_;
473 MTLSafeFreeList *active_list = get_current_safe_list();
474 /* Check while locked, multiple threads may begin a new list at the same time. */
475 if (only_if_should_flush && !active_list->should_flush()) {
476 safelist_lock_.unlock();
477 return;
478 }
479 MTLSafeFreeList *new_list = new MTLSafeFreeList();
480 if (active_list) {
481 /* Released by the active list once it is released, see #MTLSafeFreeList::next_list_. */
482 new_list->reference_count_++;
483 active_list->next_list_ = new_list;
484 }
485 current_free_list_ = new_list;
486 prev_free_buffer_list_ = active_list;
487 safelist_lock_.unlock();
488
489 /* Release final reference for previous list.
490 * NOTE: Outside of lock as this function itself locks. */
491 if (previous_list) {
492 previous_list->decrement_reference();
493 }
494}
495
496void MTLBufferPool::ensure_buffer_pool(MTLResourceOptions options)
497{
498 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.lookup_ptr(
500 if (pool_search == nullptr) {
501 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool =
502 new std::multiset<MTLBufferHandle, CompareMTLBuffer>();
503 buffer_pools_.add_new((uint64_t)options, pool);
504 }
505}
506
507void MTLBufferPool::insert_buffer_into_pool(MTLResourceOptions options, gpu::MTLBuffer *buffer)
508{
509 /* Ensure `safelist_lock_` is locked in calling code before modifying. */
510 BLI_assert(buffer);
511
512 /* Reset usage size to actual size of allocation. */
513 buffer->set_usage_size(buffer->get_size());
514
515 /* Ensure pool exists. */
516 this->ensure_buffer_pool(options);
517
518 /* TODO(Metal): Support purgeability - Allow buffer in pool to have its memory taken back by the
519 * OS if needed. As we keep allocations around, they may not actually be in use, but we can
520 * ensure they do not block other apps from using memory. Upon a buffer being needed again, we
521 * can reset this state.
522 * TODO(Metal): Purgeability state does not update instantly, so this requires a deferral. */
523 BLI_assert(buffer->get_metal_buffer());
524 // buffer->metal_buffer); [buffer->metal_buffer setPurgeableState:MTLPurgeableStateVolatile];
525
526 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool = buffer_pools_.lookup(options);
527 pool->insert(MTLBufferHandle(buffer));
528 allocations_in_pool_ += buffer->get_size();
529
530#if MTL_DEBUG_MEMORY_STATISTICS == 1
531 /* Debug statistics. */
532 buffers_in_pool_++;
533#endif
534}
535
536void MTLBufferPool::allocations_list_insert(gpu::MTLBuffer *buffer)
537{
538 /* NOTE: Function should only be called while buffer_pool_lock_ is acquired. */
539 BLI_assert(initialized_);
540 BLI_assert(buffer != nullptr);
541
542 /* Insert buffer at base of allocations list. */
543 gpu::MTLBuffer *current_head = allocations_list_base_;
544 buffer->next = current_head;
545 buffer->prev = nullptr;
546
547 if (current_head != nullptr) {
548 current_head->prev = buffer;
549 }
550
551 allocations_list_base_ = buffer;
552 allocations_list_size_++;
553
554#if MTL_DEBUG_MEMORY_STATISTICS == 1
555 total_allocation_bytes_ += buffer->get_size();
556#endif
557}
558
559void MTLBufferPool::allocations_list_delete(gpu::MTLBuffer *buffer)
560{
561 /* NOTE: Function should only be called while buffer_pool_lock_ is acquired. */
562 /* Remove a buffer link in the allocations chain. */
563 BLI_assert(initialized_);
564 BLI_assert(buffer != nullptr);
565 BLI_assert(allocations_list_size_ >= 1);
566
567 gpu::MTLBuffer *next = buffer->next;
568 gpu::MTLBuffer *prev = buffer->prev;
569
570 if (prev != nullptr) {
571 BLI_assert(prev->next == buffer);
572 prev->next = next;
573 }
574
575 if (next != nullptr) {
576 BLI_assert(next->prev == buffer);
577 next->prev = prev;
578 }
579
580 if (allocations_list_base_ == buffer) {
581 allocations_list_base_ = next;
582 BLI_assert(prev == nullptr);
583 }
584 allocations_list_size_--;
585
586#if MTL_DEBUG_MEMORY_STATISTICS == 1
587 total_allocation_bytes_ -= buffer->get_size();
588#endif
589
590 /* Delete buffer. */
591 delete buffer;
592}
593
594void MTLBufferPool::allocations_list_delete_all()
595{
596 gpu::MTLBuffer *current = allocations_list_base_;
597 while (current != nullptr) {
598 gpu::MTLBuffer *next = current->next;
599 delete current;
600 current = next;
601 }
602 allocations_list_size_ = 0;
603 allocations_list_base_ = nullptr;
604
605#if MTL_DEBUG_MEMORY_STATISTICS == 1
606 total_allocation_bytes_ = 0;
607#endif
608}
609
611{
612 reference_count_ = 1;
613 in_free_queue_ = false;
614 current_list_index_ = 0;
615 next_ = nullptr;
616 referenced_by_workload_ = false;
617}
618
620{
621 BLI_assert(in_free_queue_ == false);
622 freed_size_ += buffer->get_size();
623
624 /* Lockless list insert. */
625 uint insert_index = current_list_index_++;
626
627 /* If the current MTLSafeFreeList size is exceeded, we ripple down the linked-list chain and
628 * insert the buffer into the next available chunk. */
629 if (insert_index >= MTLSafeFreeList::MAX_NUM_BUFFERS_) {
630
631 /* Check if first caller to generate next pool in chain.
632 * Otherwise, ensure pool exists or wait for first caller to create next pool. */
633 MTLSafeFreeList *next_list = next_.load();
634
635 if (!next_list) {
636 std::unique_lock lock(lock_);
637
638 next_list = next_.load();
639 if (!next_list) {
640 next_list = new MTLSafeFreeList();
641 next_.store(next_list);
642 }
643 }
644 BLI_assert(next_list);
645 next_list->insert_buffer(buffer);
646
647 /* Clamp index to chunk limit if overflowing. */
648 current_list_index_ = MTLSafeFreeList::MAX_NUM_BUFFERS_;
649 return;
650 }
651
652 safe_free_pool_[insert_index] = buffer;
653}
654
655/* Increments from active GPUContext thread. */
657{
658 lock_.lock();
659 BLI_assert(in_free_queue_ == false);
660 reference_count_++;
661 referenced_by_workload_ = true;
662 lock_.unlock();
663}
664
665/* Reference decrements and addition to completed list queue can occur from MTLCommandBuffer
666 * completion callback thread. */
668{
669 /* Releasing a list releases the reference it holds on the next list. */
670 MTLSafeFreeList *list = this;
671 while (list) {
672 list->lock_.lock();
673 BLI_assert(list->in_free_queue_ == false);
674 int ref_count = --list->reference_count_;
675
676 MTLSafeFreeList *next_list = nullptr;
677 if (ref_count == 0) {
678 next_list = list->next_list_;
680 }
681 list->lock_.unlock();
682 list = next_list;
683 }
684}
685
687{
688 /* We should only consider refreshing a list if it has been referenced by active workloads, and
689 * contains a sufficient buffer count or size to avoid overheads associated with flushing the
690 * list. If the list hasn't been referenced by a command buffer, buffers may have been added, but
691 * no command submissions will have been issued, hence buffers could be returned to the pool
692 * prematurely if associated workload submission occurs later. */
693 return (referenced_by_workload_ &&
694 (current_list_index_ > MIN_BUFFER_FLUSH_COUNT || freed_size_ >= MIN_FLUSH_SIZE));
695}
696
698
699/* -------------------------------------------------------------------- */
702
704 uint64_t size,
705 MTLResourceOptions options,
706 uint alignment)
707{
708 /* Calculate aligned allocation size. */
709 BLI_assert(alignment > 0);
710 uint64_t aligned_alloc_size = ceil_to_multiple_ul(size, alignment);
711
712 alignment_ = alignment;
713 device_ = mtl_device;
714 is_external_ = false;
715
716 options_ = options;
717 this->flag_in_use(false);
718
719 metal_buffer_ = [device_ newBufferWithLength:aligned_alloc_size options:options];
720 BLI_assert(metal_buffer_);
721
722 size_ = aligned_alloc_size;
723 this->set_usage_size(size_);
724 if (!(options_ & MTLResourceStorageModePrivate)) {
725 data_ = [metal_buffer_ contents];
726 }
727 else {
728 data_ = nullptr;
729 }
730
731 /* Linked resources. */
732 next = prev = nullptr;
733}
734
736{
737 BLI_assert(external_buffer != nil);
738
739 /* Ensure external_buffer remains referenced while in-use. */
740 metal_buffer_ = external_buffer;
741 [metal_buffer_ retain];
742
743 /* Extract properties. */
744 is_external_ = true;
745 device_ = nil;
746 alignment_ = 1;
747 options_ = [metal_buffer_ resourceOptions];
748 size_ = [metal_buffer_ allocatedSize];
749 this->set_usage_size(size_);
750 data_ = [metal_buffer_ contents];
751 in_use_ = true;
752
753 /* Linked resources. */
754 next = prev = nullptr;
755}
756
758{
759 if (metal_buffer_ != nil) {
760 [metal_buffer_ release];
761 metal_buffer_ = nil;
762 }
763}
764
766{
767 if (!is_external_) {
769 }
770 else {
771 if (metal_buffer_ != nil) {
772 [metal_buffer_ release];
773 metal_buffer_ = nil;
774 }
775 }
776}
777
779{
780 return metal_buffer_;
781}
782
784{
785 BLI_assert(!(options_ & MTLResourceStorageModePrivate));
786 BLI_assert(data_);
787 return data_;
788}
789
791{
792 return size_;
793}
794
796{
797 return usage_size_;
798}
799
801{
802 /* We do not need to flush shared memory, as addressable buffer is shared. */
803 return options_ & MTLResourceStorageModeManaged;
804}
805
807{
808 metal_buffer_.label = str;
809}
810
812{
813 /* Debug: If buffer is not flagged as in-use, this is a problem. */
815 in_use_,
816 "Buffer should be marked as 'in-use' if being actively used by an instance. Buffer "
817 "has likely already been freed.");
818}
819
821{
822 this->debug_ensure_used();
823 if (this->requires_flush()) {
824 [metal_buffer_ didModifyRange:NSMakeRange(0, size_)];
825 }
826}
827
829{
830 this->debug_ensure_used();
831 if (this->requires_flush()) {
832 BLI_assert((offset + length) <= size_);
833 [metal_buffer_ didModifyRange:NSMakeRange(offset, length)];
834 }
835}
836
838{
839 in_use_ = used;
840}
841
843{
844 return in_use_;
845}
846
848{
849 BLI_assert(size_used > 0 && size_used <= size_);
850 usage_size_ = size_used;
851}
852
854{
855 return options_;
856}
857
859{
860 return alignment_;
861}
862
864{
865 /* We do not need to flush shared memory. */
866 return this->options & MTLResourceStorageModeManaged;
867}
868
870{
871 if (this->requires_flush()) {
873 BLI_assert((this->buffer_offset + this->size) <= [this->metal_buffer length]);
874 BLI_assert(this->buffer_offset >= 0);
875 [this->metal_buffer
876 didModifyRange:NSMakeRange(this->buffer_offset, this->size - this->buffer_offset)];
877 }
878}
879
881
882/* -------------------------------------------------------------------- */
885
890
892{
893
894 if (!this->initialised_) {
895 BLI_assert(context_.device);
896
897 /* Initialize Scratch buffers. */
898 for (int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
899 scratch_buffers_[sb] = new MTLCircularBuffer(
900 context_, mtl_scratch_buffer_initial_size_, true);
901 BLI_assert(scratch_buffers_[sb]);
902 BLI_assert(&(scratch_buffers_[sb]->own_context_) == &context_);
903 }
904 current_scratch_buffer_ = 0;
905 initialised_ = true;
906 }
907}
908
910{
911 initialised_ = false;
912
913 /* Release Scratch buffers */
914 for (int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
915 delete scratch_buffers_[sb];
916 scratch_buffers_[sb] = nullptr;
917 }
918 current_scratch_buffer_ = 0;
919}
920
925
927 uint64_t alloc_size, uint alignment)
928{
929 /* Ensure scratch buffer allocation alignment adheres to offset alignment requirements. */
930 alignment = max_uu(alignment, 256);
931
932 BLI_assert_msg(current_scratch_buffer_ >= 0, "Scratch Buffer index not set");
933 MTLCircularBuffer *current_scratch_buff = this->scratch_buffers_[current_scratch_buffer_];
934 BLI_assert_msg(current_scratch_buff != nullptr, "Scratch Buffer does not exist");
935 MTLTemporaryBuffer allocated_range = current_scratch_buff->allocate_range_aligned(alloc_size,
936 alignment);
937 BLI_assert(allocated_range.size >= alloc_size && allocated_range.size <= alloc_size + alignment);
938 BLI_assert(allocated_range.metal_buffer != nil);
939 return allocated_range;
940}
941
943{
944 /* Fetch active scratch buffer. */
945 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
946 BLI_assert(&active_scratch_buf->own_context_ == &context_);
947
948 /* Ensure existing scratch buffer is no longer in use. MTL_MAX_SCRATCH_BUFFERS specifies
949 * the number of allocated scratch buffers. This value should be equal to the number of
950 * simultaneous frames in-flight. I.e. the maximal number of scratch buffers which are
951 * simultaneously in-use. */
952 if (active_scratch_buf->used_frame_index_ < context_.get_current_frame_index()) {
953 current_scratch_buffer_ = (current_scratch_buffer_ + 1) % mtl_max_scratch_buffers_;
954 active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
955 active_scratch_buf->reset();
956 BLI_assert(&active_scratch_buf->own_context_ == &context_);
957 MTL_LOG_DEBUG("Scratch buffer %d reset - (ctx %p)(Frame index: %d)",
958 current_scratch_buffer_,
959 &context_,
960 context_.get_current_frame_index());
961 }
962}
963
965{
966 /* Fetch active scratch buffer and verify context. */
967 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
968 BLI_assert(&active_scratch_buf->own_context_ == &context_);
969 active_scratch_buf->flush();
970}
971
973{
974 /* Fetch active scratch buffer and verify context. */
975 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
976 BLI_assert(&active_scratch_buf->own_context_ == &context_);
977 active_scratch_buf->ssbo_source_->bind(slot);
978}
979
981{
982 /* Fetch active scratch buffer and verify context. */
983 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
984 BLI_assert(&active_scratch_buf->own_context_ == &context_);
985 active_scratch_buf->ssbo_source_->unbind();
986}
987
988/* MTLCircularBuffer implementation. */
989MTLCircularBuffer::MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow)
990 : own_context_(ctx)
991{
992 BLI_assert(this);
993 ssbo_source_ = new gpu::MTLStorageBuf(initial_size);
994 cbuffer_ = ssbo_source_->metal_buffer_;
995 current_offset_ = 0;
996 can_resize_ = allow_grow;
997 cbuffer_->flag_in_use(true);
998
999 used_frame_index_ = ctx.get_current_frame_index();
1000 last_flush_base_offset_ = 0;
1001
1002 /* Debug label. */
1003 if (G.debug & G_DEBUG_GPU) {
1004 cbuffer_->set_label(@"Circular Scratch Buffer");
1005 }
1006}
1007
1009{
1010 delete ssbo_source_;
1011}
1012
1014{
1015 return this->allocate_range_aligned(alloc_size, 1);
1016}
1017
1019{
1020 BLI_assert(this);
1021
1022 /* Ensure alignment of an allocation is aligned to compatible offset boundaries. */
1023 BLI_assert(alignment > 0);
1024 alignment = max_uu(alignment, 256);
1025
1026 /* Align current offset and allocation size to desired alignment */
1027 uint64_t aligned_current_offset = ceil_to_multiple_ul(current_offset_, alignment);
1028 uint64_t aligned_alloc_size = ceil_to_multiple_ul(alloc_size, alignment);
1029 bool can_allocate = (aligned_current_offset + aligned_alloc_size) < cbuffer_->get_size();
1030
1031 BLI_assert(aligned_current_offset >= current_offset_);
1032 BLI_assert(aligned_alloc_size >= alloc_size);
1033
1034 BLI_assert(aligned_current_offset % alignment == 0);
1035 BLI_assert(aligned_alloc_size % alignment == 0);
1036
1037 /* Recreate Buffer */
1038 if (!can_allocate) {
1039 uint64_t new_size = cbuffer_->get_size();
1040 if (can_resize_) {
1041 /* Resize to the maximum of basic resize heuristic OR the size of the current offset +
1042 * requested allocation -- we want the buffer to grow to a large enough size such that it
1043 * does not need to resize mid-frame. */
1045 ceil_to_multiple_ul(new_size * 1.2f, 256)),
1046 aligned_current_offset + aligned_alloc_size);
1047
1048#ifdef MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION
1049 /* IF a requested allocation EXCEEDS the maximum supported size, temporarily allocate up to
1050 * this, but shrink down ASAP. */
1052
1053 /* If new requested allocation is bigger than maximum allowed size, temporarily resize to
1054 * maximum allocation size -- Otherwise, clamp the buffer size back down to the defined
1055 * maximum */
1057 new_size = aligned_alloc_size;
1058 MTL_LOG_DEBUG("Temporarily growing Scratch buffer to %d MB",
1059 (int)new_size / 1024 / 1024);
1060 }
1061 else {
1063 MTL_LOG_DEBUG("Shrinking Scratch buffer back to %d MB", (int)new_size / 1024 / 1024);
1064 }
1065 }
1066 BLI_assert(aligned_alloc_size <= new_size);
1067#else
1069
1070 if (aligned_alloc_size > new_size) {
1071 BLI_assert(false);
1072
1073 /* Cannot allocate */
1074 MTLTemporaryBuffer alloc_range;
1075 alloc_range.metal_buffer = nil;
1076 alloc_range.data = nullptr;
1077 alloc_range.buffer_offset = 0;
1078 alloc_range.size = 0;
1079 alloc_range.options = cbuffer_->options;
1080 }
1081#endif
1082 }
1083 else {
1085 "Performance Warning: Reached the end of circular buffer of size: %llu, but cannot "
1086 "resize. Starting new buffer",
1087 cbuffer_->get_size());
1088 BLI_assert(aligned_alloc_size <= new_size);
1089
1090 /* Cannot allocate. */
1091 MTLTemporaryBuffer alloc_range;
1092 alloc_range.metal_buffer = nil;
1093 alloc_range.data = nullptr;
1094 alloc_range.buffer_offset = 0;
1095 alloc_range.size = 0;
1096 alloc_range.options = cbuffer_->get_resource_options();
1097 }
1098
1099 /* Flush current buffer to ensure changes are visible on the GPU. */
1100 this->flush();
1101
1102 /* Discard old buffer and create a new one - Relying on Metal reference counting to track
1103 * in-use buffers */
1104 delete ssbo_source_;
1105 ssbo_source_ = new gpu::MTLStorageBuf(new_size);
1106 cbuffer_ = ssbo_source_->metal_buffer_;
1107 cbuffer_->flag_in_use(true);
1108 current_offset_ = 0;
1109 last_flush_base_offset_ = 0;
1110
1111 /* Debug label. */
1112 if (G.debug & G_DEBUG_GPU) {
1113 cbuffer_->set_label(@"Circular Scratch Buffer");
1114 }
1115 MTL_LOG_DEBUG("Resized Metal circular buffer to %llu bytes", new_size);
1116
1117 /* Reset allocation Status. */
1118 aligned_current_offset = 0;
1119 BLI_assert((aligned_current_offset + aligned_alloc_size) <= cbuffer_->get_size());
1120 }
1121
1122 /* Allocate chunk. */
1123 MTLTemporaryBuffer alloc_range;
1124 alloc_range.metal_buffer = cbuffer_->get_metal_buffer();
1125 alloc_range.data = (void *)((uint8_t *)([alloc_range.metal_buffer contents]) +
1126 aligned_current_offset);
1127 alloc_range.buffer_offset = aligned_current_offset;
1128 alloc_range.size = aligned_alloc_size;
1129 alloc_range.options = cbuffer_->get_resource_options();
1130 BLI_assert(alloc_range.data);
1131
1132 /* Shift offset to match alignment. */
1133 current_offset_ = aligned_current_offset + aligned_alloc_size;
1134 BLI_assert(current_offset_ <= cbuffer_->get_size());
1135 return alloc_range;
1136}
1137
1139{
1140 BLI_assert(this);
1141
1142 uint64_t len = current_offset_ - last_flush_base_offset_;
1143 if (len > 0) {
1144 cbuffer_->flush_range(last_flush_base_offset_, len);
1145 last_flush_base_offset_ = current_offset_;
1146 }
1147}
1148
1150{
1151 BLI_assert(this);
1152
1153 /* If circular buffer has data written to it, offset will be greater than zero. */
1154 if (current_offset_ > 0) {
1155
1156 /* Ensure the circular buffer is no longer being used by an in-flight frame. */
1157 BLI_assert((own_context_.get_current_frame_index() >=
1158 (used_frame_index_ + MTL_NUM_SAFE_FRAMES - 1)) &&
1159 "Trying to reset Circular scratch buffer's while its data is still being used by "
1160 "an in-flight frame");
1161
1162 current_offset_ = 0;
1163 last_flush_base_offset_ = 0;
1164 }
1165
1166 /* Update used frame index to current. */
1167 used_frame_index_ = own_context_.get_current_frame_index();
1168}
1169
1171
1172} // namespace blender::gpu
#define CLOG_DEBUG(clg_ref,...)
Definition CLG_log.h:191
volatile int lock
#define BLI_assert(a)
#define BLI_assert_msg(a, msg)
const Value * lookup_ptr(const Key &key) const
Definition BLI_map.hh:508
const Value & lookup(const Key &key) const
Definition BLI_map.hh:545
void add_new(const Key &key, const Value &value)
Definition BLI_map.hh:265
void push_completed_safe_list(MTLSafeFreeList *list)
void init(id< MTLDevice > device)
Definition mtl_memory.mm:40
gpu::MTLBuffer * allocate_with_data(uint64_t size, bool cpu_visible, const void *data=nullptr)
MTLSafeFreeList * acquire_current_safe_list()
MTLSafeFreeList * get_current_safe_list()
gpu::MTLBuffer * allocate(uint64_t size, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned(uint64_t size, uint alignment, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned_with_data(uint64_t size, uint alignment, bool cpu_visible, const void *data=nullptr)
bool free_buffer(gpu::MTLBuffer *buffer)
void begin_new_safe_list(bool only_if_should_flush=false)
void flag_in_use(bool used)
uint64_t get_size() const
void set_usage_size(uint64_t size_used)
gpu::MTLBuffer * next
uint64_t get_size_used() const
void * get_host_ptr() const
void flush_range(uint64_t offset, uint64_t length)
MTLBuffer(id< MTLDevice > device, uint64_t size, MTLResourceOptions options, uint alignment=1)
gpu::MTLBuffer * prev
void set_label(NSString *str)
MTLResourceOptions get_resource_options()
id< MTLBuffer > get_metal_buffer() const
MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow)
MTLTemporaryBuffer allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer allocate_range(uint64_t alloc_size)
static MTLBufferPool * get_global_memory_manager()
void insert_buffer(gpu::MTLBuffer *buffer)
static constexpr uint mtl_scratch_buffer_max_size_
static constexpr uint mtl_scratch_buffer_initial_size_
MTLTemporaryBuffer scratch_buffer_allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer scratch_buffer_allocate_range(uint64_t alloc_size)
void bind(int slot) override
CCL_NAMESPACE_BEGIN struct Options options
#define str(s)
float length(VecOp< float, D >) RET
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
Definition math_dual.h:263
#define MTL_NUM_SAFE_FRAMES
Definition mtl_common.hh:19
#define MTL_LOG_WARNING(info,...)
Definition mtl_debug.hh:39
#define MTL_LOG_DEBUG(info,...)
Definition mtl_debug.hh:46
static constexpr size_t MEMORY_SIZE_1GB
Definition mtl_memory.mm:29
static constexpr size_t MEMORY_SIZE_512MB
Definition mtl_memory.mm:28
static constexpr size_t MEMORY_SIZE_2GB
Definition mtl_memory.mm:30
static constexpr size_t MEMORY_SIZE_256MB
Definition mtl_memory.mm:27
static CLG_LogRef LOG
Definition mtl_memory.mm:34
MTLBufferRange MTLTemporaryBuffer
SymEdge< T > * prev(const SymEdge< T > *se)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
MINLINE unsigned long long max_ulul(unsigned long long a, unsigned long long b)
MINLINE unsigned long long min_ulul(unsigned long long a, unsigned long long b)
unsigned __int64 uint64_t
Definition BKE_text.h:91
unsigned int uint
Definition BKE_text.h:55
MINLINE uint max_uu(uint a, uint b)
Global G
Definition blender.cc:57
unsigned int uint32_t
Definition BKE_text.h:81
MINLINE uint64_t ceil_to_multiple_ul(uint64_t a, uint64_t b)
BMesh const char void * data
unsigned char uint8_t
Definition BKE_text.h:79
MINLINE int min_ii(int a, int b)
MTLBufferHandle(gpu::MTLBuffer *buf)
MTLResourceOptions options