24#define MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION
47#if MTL_DEBUG_MEMORY_STATISTICS == 1
49 total_allocation_bytes_ = 0;
50 per_frame_allocation_count_ = 0;
54 allocations_in_pool_ = 0;
57 allocations_list_base_ =
nullptr;
58 allocations_list_size_ = 0;
71void MTLBufferPool::free()
73 buffer_pool_lock_.lock();
76 allocations_list_delete_all();
79 for (
int safe_pool_free_index = 0; safe_pool_free_index < completed_safelist_queue_.size();
80 safe_pool_free_index++)
82 delete completed_safelist_queue_[safe_pool_free_index];
84 completed_safelist_queue_.clear();
86 safelist_lock_.lock();
87 if (current_free_list_ !=
nullptr) {
88 delete current_free_list_;
89 current_free_list_ =
nullptr;
91 if (prev_free_buffer_list_ !=
nullptr) {
92 delete prev_free_buffer_list_;
93 prev_free_buffer_list_ =
nullptr;
95 safelist_lock_.unlock();
98 for (std::multiset<blender::gpu::MTLBufferHandle, blender::gpu::CompareMTLBuffer> *buffer_pool :
99 buffer_pools_.values())
104 buffer_pools_.clear();
105 buffer_pool_lock_.unlock();
138 options = ([device_ hasUnifiedMemory]) ? MTLResourceStorageModeShared :
139 MTLResourceStorageModeManaged;
142 options = MTLResourceStorageModePrivate;
147 buffer_pool_lock_.lock();
149 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.lookup_ptr(
152 if (pool_search !=
nullptr) {
153 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool = *pool_search;
155 auto result = pool->lower_bound(size_compare);
156 if (
result != pool->end()) {
164 if (found_size >= aligned_alloc_size &&
165 found_size <= (aligned_alloc_size * mtl_buffer_size_threshold_factor_))
168 "[MemoryAllocator] Suitable Buffer of size %lld found, for requested size: %lld",
172 new_buffer = found_buffer;
180 "[MemoryAllocator] Buffer of size %lld found, but was incompatible with requested "
184 new_buffer =
nullptr;
190 if (new_buffer ==
nullptr) {
194 allocations_list_insert(new_buffer);
200#if MTL_DEBUG_MEMORY_STATISTICS == 1
207 allocations_in_pool_ -= new_buffer->
get_size();
215#if MTL_DEBUG_MEMORY_STATISTICS == 1
216 per_frame_allocation_count_++;
220 buffer_pool_lock_.unlock();
253 safelist_lock_.lock();
256 current_pool->reference_count_++;
257 safelist_lock_.unlock();
278 safelist_lock_.lock();
279 buffer_pool_lock_.lock();
281#if MTL_DEBUG_MEMORY_STATISTICS == 1
282 int num_buffers_added = 0;
286 for (
int safe_pool_free_index = 0; safe_pool_free_index < completed_safelist_queue_.size();
287 safe_pool_free_index++)
289 MTLSafeFreeList *current_pool = completed_safelist_queue_[safe_pool_free_index];
292 while (current_pool !=
nullptr) {
293 current_pool->lock_.lock();
297 int size =
min_ii(current_pool->current_list_index_, MTLSafeFreeList::MAX_NUM_BUFFERS_);
300 while (counter < size) {
309#if MTL_DEBUG_MEMORY_STATISTICS == 1
318 current_pool->lock_.unlock();
320 current_pool =
nullptr;
323 if (next_list !=
nullptr) {
324 current_pool = next_list;
335 const time_t time_now = std::time(
nullptr);
336 for (
auto buffer_pool_list : buffer_pools_.items()) {
337 MTLBufferPoolOrderedList *pool_allocations = buffer_pool_list.value;
338 MTLBufferPoolOrderedList::iterator pool_iterator = pool_allocations->begin();
339 while (pool_iterator != pool_allocations->end()) {
342 const time_t time_passed = time_now - handle.
insert_time;
347 time_t deletion_time_threshold_s = 600;
350 deletion_time_threshold_s = 2;
355 deletion_time_threshold_s = 4;
359 deletion_time_threshold_s = 15;
363 deletion_time_threshold_s = 60;
366 if (time_passed > deletion_time_threshold_s) {
369 allocations_list_delete(handle.
buffer);
372 pool_iterator = pool_allocations->erase(pool_iterator);
374#if MTL_DEBUG_MEMORY_STATISTICS == 1
383#if MTL_DEBUG_MEMORY_STATISTICS == 1
385 CLOG_DEBUG(&
LOG,
" Num buffers processed in pool (this frame): %u", num_buffers_added);
387 uint framealloc = (
uint)per_frame_allocation_count_;
389 CLOG_DEBUG(&
LOG,
" Total Buffers allocated: %u", allocations_list_size_);
391 &
LOG,
" Total Memory allocated: %u MB", (
uint)total_allocation_bytes_ / (1024 * 1024));
393 uint allocs = (
uint)(allocations_in_pool_) / 1024 / 2024;
396 uint buffs = (
uint)buffers_in_pool_;
400 auto key_iterator = buffer_pools_.keys().begin();
401 auto value_iterator = buffer_pools_.values().begin();
402 while (key_iterator != buffer_pools_.keys().end()) {
405 for (
auto it = (*value_iterator)->begin(); it != (*value_iterator)->end(); it++) {
406 mem_in_pool += it->buffer_size;
411 " Buffers in pool (%u)(%llu): %u (%u MB)",
414 (
uint)((*value_iterator)->size()),
415 (
uint)mem_in_pool / 1024 / 1024);
420 per_frame_allocation_count_ = 0;
424 completed_safelist_queue_.clear();
425 buffer_pool_lock_.unlock();
426 safelist_lock_.unlock();
434 safe_list->lock_.lock();
436 BLI_assert(safe_list->reference_count_ == 0 &&
437 "Pool must be fully dereferenced by all in-use cmd buffers before returning.\n");
438 BLI_assert(safe_list->in_free_queue_ ==
false &&
"Pool must not already be in queue");
442 safelist_lock_.lock();
443 completed_safelist_queue_.append(safe_list);
444 safelist_lock_.unlock();
445 safe_list->lock_.unlock();
451 return current_free_list_;
457 std::scoped_lock
lock(safelist_lock_);
459 list->increment_reference();
465 std::scoped_lock
lock(safelist_lock_);
466 return !completed_safelist_queue_.is_empty();
471 safelist_lock_.lock();
475 if (only_if_should_flush && !active_list->
should_flush()) {
476 safelist_lock_.unlock();
482 new_list->reference_count_++;
483 active_list->next_list_ = new_list;
485 current_free_list_ = new_list;
486 prev_free_buffer_list_ = active_list;
487 safelist_lock_.unlock();
496void MTLBufferPool::ensure_buffer_pool(MTLResourceOptions
options)
498 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.
lookup_ptr(
500 if (pool_search ==
nullptr) {
501 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool =
502 new std::multiset<MTLBufferHandle, CompareMTLBuffer>();
516 this->ensure_buffer_pool(
options);
526 std::multiset<MTLBufferHandle, CompareMTLBuffer> *pool = buffer_pools_.
lookup(
options);
528 allocations_in_pool_ += buffer->
get_size();
530#if MTL_DEBUG_MEMORY_STATISTICS == 1
536void MTLBufferPool::allocations_list_insert(gpu::MTLBuffer *buffer)
543 gpu::MTLBuffer *current_head = allocations_list_base_;
544 buffer->
next = current_head;
545 buffer->
prev =
nullptr;
547 if (current_head !=
nullptr) {
548 current_head->
prev = buffer;
551 allocations_list_base_ = buffer;
552 allocations_list_size_++;
554#if MTL_DEBUG_MEMORY_STATISTICS == 1
555 total_allocation_bytes_ += buffer->
get_size();
559void MTLBufferPool::allocations_list_delete(gpu::MTLBuffer *buffer)
567 gpu::MTLBuffer *next = buffer->
next;
568 gpu::MTLBuffer *
prev = buffer->
prev;
570 if (prev !=
nullptr) {
575 if (next !=
nullptr) {
580 if (allocations_list_base_ == buffer) {
581 allocations_list_base_ = next;
584 allocations_list_size_--;
586#if MTL_DEBUG_MEMORY_STATISTICS == 1
587 total_allocation_bytes_ -= buffer->
get_size();
594void MTLBufferPool::allocations_list_delete_all()
596 gpu::MTLBuffer *current = allocations_list_base_;
597 while (current !=
nullptr) {
598 gpu::MTLBuffer *next = current->
next;
602 allocations_list_size_ = 0;
603 allocations_list_base_ =
nullptr;
605#if MTL_DEBUG_MEMORY_STATISTICS == 1
606 total_allocation_bytes_ = 0;
612 reference_count_ = 1;
613 in_free_queue_ =
false;
614 current_list_index_ = 0;
616 referenced_by_workload_ =
false;
625 uint insert_index = current_list_index_++;
629 if (insert_index >= MTLSafeFreeList::MAX_NUM_BUFFERS_) {
636 std::unique_lock
lock(lock_);
638 next_list = next_.load();
641 next_.store(next_list);
648 current_list_index_ = MTLSafeFreeList::MAX_NUM_BUFFERS_;
652 safe_free_pool_[insert_index] = buffer;
661 referenced_by_workload_ =
true;
674 int ref_count = --
list->reference_count_;
677 if (ref_count == 0) {
678 next_list =
list->next_list_;
681 list->lock_.unlock();
693 return (referenced_by_workload_ &&
694 (current_list_index_ > MIN_BUFFER_FLUSH_COUNT || freed_size_ >= MIN_FLUSH_SIZE));
712 alignment_ = alignment;
713 device_ = mtl_device;
714 is_external_ =
false;
719 metal_buffer_ = [device_ newBufferWithLength:aligned_alloc_size
options:
options];
722 size_ = aligned_alloc_size;
724 if (!(options_ & MTLResourceStorageModePrivate)) {
725 data_ = [metal_buffer_ contents];
740 metal_buffer_ = external_buffer;
741 [metal_buffer_ retain];
747 options_ = [metal_buffer_ resourceOptions];
748 size_ = [metal_buffer_ allocatedSize];
750 data_ = [metal_buffer_ contents];
759 if (metal_buffer_ != nil) {
760 [metal_buffer_ release];
771 if (metal_buffer_ != nil) {
772 [metal_buffer_ release];
780 return metal_buffer_;
785 BLI_assert(!(options_ & MTLResourceStorageModePrivate));
803 return options_ & MTLResourceStorageModeManaged;
808 metal_buffer_.label =
str;
816 "Buffer should be marked as 'in-use' if being actively used by an instance. Buffer "
817 "has likely already been freed.");
824 [metal_buffer_ didModifyRange:NSMakeRange(0, size_)];
833 [metal_buffer_ didModifyRange:NSMakeRange(offset,
length)];
849 BLI_assert(size_used > 0 && size_used <= size_);
850 usage_size_ = size_used;
866 return this->options & MTLResourceStorageModeManaged;
894 if (!this->initialised_) {
898 for (
int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
902 BLI_assert(&(scratch_buffers_[sb]->own_context_) == &context_);
904 current_scratch_buffer_ = 0;
911 initialised_ =
false;
914 for (
int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
915 delete scratch_buffers_[sb];
916 scratch_buffers_[sb] =
nullptr;
918 current_scratch_buffer_ = 0;
930 alignment =
max_uu(alignment, 256);
932 BLI_assert_msg(current_scratch_buffer_ >= 0,
"Scratch Buffer index not set");
933 MTLCircularBuffer *current_scratch_buff = this->scratch_buffers_[current_scratch_buffer_];
934 BLI_assert_msg(current_scratch_buff !=
nullptr,
"Scratch Buffer does not exist");
937 BLI_assert(allocated_range.
size >= alloc_size && allocated_range.
size <= alloc_size + alignment);
939 return allocated_range;
945 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
946 BLI_assert(&active_scratch_buf->own_context_ == &context_);
952 if (active_scratch_buf->used_frame_index_ < context_.get_current_frame_index()) {
953 current_scratch_buffer_ = (current_scratch_buffer_ + 1) % mtl_max_scratch_buffers_;
954 active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
955 active_scratch_buf->
reset();
956 BLI_assert(&active_scratch_buf->own_context_ == &context_);
957 MTL_LOG_DEBUG(
"Scratch buffer %d reset - (ctx %p)(Frame index: %d)",
958 current_scratch_buffer_,
960 context_.get_current_frame_index());
967 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
968 BLI_assert(&active_scratch_buf->own_context_ == &context_);
969 active_scratch_buf->
flush();
975 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
976 BLI_assert(&active_scratch_buf->own_context_ == &context_);
977 active_scratch_buf->ssbo_source_->
bind(slot);
983 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
984 BLI_assert(&active_scratch_buf->own_context_ == &context_);
985 active_scratch_buf->ssbo_source_->
unbind();
994 cbuffer_ = ssbo_source_->metal_buffer_;
996 can_resize_ = allow_grow;
997 cbuffer_->flag_in_use(
true);
1000 last_flush_base_offset_ = 0;
1004 cbuffer_->set_label(
@"Circular Scratch Buffer");
1010 delete ssbo_source_;
1024 alignment =
max_uu(alignment, 256);
1029 bool can_allocate = (aligned_current_offset + aligned_alloc_size) < cbuffer_->get_size();
1031 BLI_assert(aligned_current_offset >= current_offset_);
1032 BLI_assert(aligned_alloc_size >= alloc_size);
1034 BLI_assert(aligned_current_offset % alignment == 0);
1035 BLI_assert(aligned_alloc_size % alignment == 0);
1038 if (!can_allocate) {
1039 uint64_t new_size = cbuffer_->get_size();
1046 aligned_current_offset + aligned_alloc_size);
1048#ifdef MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION
1057 new_size = aligned_alloc_size;
1058 MTL_LOG_DEBUG(
"Temporarily growing Scratch buffer to %d MB",
1059 (
int)new_size / 1024 / 1024);
1063 MTL_LOG_DEBUG(
"Shrinking Scratch buffer back to %d MB", (
int)new_size / 1024 / 1024);
1070 if (aligned_alloc_size > new_size) {
1076 alloc_range.
data =
nullptr;
1078 alloc_range.
size = 0;
1079 alloc_range.
options = cbuffer_->options;
1085 "Performance Warning: Reached the end of circular buffer of size: %llu, but cannot "
1086 "resize. Starting new buffer",
1087 cbuffer_->get_size());
1093 alloc_range.
data =
nullptr;
1095 alloc_range.
size = 0;
1096 alloc_range.
options = cbuffer_->get_resource_options();
1104 delete ssbo_source_;
1106 cbuffer_ = ssbo_source_->metal_buffer_;
1107 cbuffer_->flag_in_use(
true);
1108 current_offset_ = 0;
1109 last_flush_base_offset_ = 0;
1113 cbuffer_->set_label(
@"Circular Scratch Buffer");
1115 MTL_LOG_DEBUG(
"Resized Metal circular buffer to %llu bytes", new_size);
1118 aligned_current_offset = 0;
1119 BLI_assert((aligned_current_offset + aligned_alloc_size) <= cbuffer_->get_size());
1124 alloc_range.
metal_buffer = cbuffer_->get_metal_buffer();
1126 aligned_current_offset);
1128 alloc_range.
size = aligned_alloc_size;
1129 alloc_range.
options = cbuffer_->get_resource_options();
1133 current_offset_ = aligned_current_offset + aligned_alloc_size;
1134 BLI_assert(current_offset_ <= cbuffer_->get_size());
1142 uint64_t len = current_offset_ - last_flush_base_offset_;
1144 cbuffer_->flush_range(last_flush_base_offset_,
len);
1145 last_flush_base_offset_ = current_offset_;
1154 if (current_offset_ > 0) {
1157 BLI_assert((own_context_.get_current_frame_index() >=
1159 "Trying to reset Circular scratch buffer's while its data is still being used by "
1160 "an in-flight frame");
1162 current_offset_ = 0;
1163 last_flush_base_offset_ = 0;
1167 used_frame_index_ = own_context_.get_current_frame_index();
#define CLOG_DEBUG(clg_ref,...)
#define BLI_assert_msg(a, msg)
const Value * lookup_ptr(const Key &key) const
const Value & lookup(const Key &key) const
void add_new(const Key &key, const Value &value)
void push_completed_safe_list(MTLSafeFreeList *list)
bool has_completed_safe_lists()
void init(id< MTLDevice > device)
gpu::MTLBuffer * allocate_with_data(uint64_t size, bool cpu_visible, const void *data=nullptr)
MTLSafeFreeList * acquire_current_safe_list()
MTLSafeFreeList * get_current_safe_list()
void update_memory_pools()
gpu::MTLBuffer * allocate(uint64_t size, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned(uint64_t size, uint alignment, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned_with_data(uint64_t size, uint alignment, bool cpu_visible, const void *data=nullptr)
bool free_buffer(gpu::MTLBuffer *buffer)
void begin_new_safe_list(bool only_if_should_flush=false)
void flag_in_use(bool used)
uint64_t get_size() const
void set_usage_size(uint64_t size_used)
uint64_t get_size_used() const
void * get_host_ptr() const
void flush_range(uint64_t offset, uint64_t length)
MTLBuffer(id< MTLDevice > device, uint64_t size, MTLResourceOptions options, uint alignment=1)
void set_label(NSString *str)
MTLResourceOptions get_resource_options()
id< MTLBuffer > get_metal_buffer() const
MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow)
MTLTemporaryBuffer allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer allocate_range(uint64_t alloc_size)
uint get_current_frame_index()
static MTLBufferPool * get_global_memory_manager()
void increment_reference()
void decrement_reference()
void insert_buffer(gpu::MTLBuffer *buffer)
static constexpr uint mtl_scratch_buffer_max_size_
static constexpr uint mtl_scratch_buffer_initial_size_
MTLTemporaryBuffer scratch_buffer_allocate_range_aligned(uint64_t alloc_size, uint alignment)
void flush_active_scratch_buffer()
void ensure_increment_scratch_buffer()
~MTLScratchBufferManager()
MTLTemporaryBuffer scratch_buffer_allocate_range(uint64_t alloc_size)
void bind_as_ssbo(int slot)
void bind(int slot) override
CCL_NAMESPACE_BEGIN struct Options options
float length(VecOp< float, D >) RET
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
#define MTL_NUM_SAFE_FRAMES
#define MTL_LOG_WARNING(info,...)
#define MTL_LOG_DEBUG(info,...)
static constexpr size_t MEMORY_SIZE_1GB
static constexpr size_t MEMORY_SIZE_512MB
static constexpr size_t MEMORY_SIZE_2GB
static constexpr size_t MEMORY_SIZE_256MB
MTLBufferRange MTLTemporaryBuffer
SymEdge< T > * prev(const SymEdge< T > *se)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
MINLINE unsigned long long max_ulul(unsigned long long a, unsigned long long b)
MINLINE unsigned long long min_ulul(unsigned long long a, unsigned long long b)
unsigned __int64 uint64_t
MINLINE uint max_uu(uint a, uint b)
MINLINE uint64_t ceil_to_multiple_ul(uint64_t a, uint64_t b)
BMesh const char void * data
MINLINE int min_ii(int a, int b)
MTLBufferHandle(gpu::MTLBuffer *buf)
MTLResourceOptions options
id< MTLBuffer > metal_buffer