Blender V5.3
mtl_memory.hh
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#pragma once
10
11#include "BLI_map.hh"
12
13#include "mtl_common.hh"
14
15#include <atomic>
16#include <ctime>
17#include <functional>
18#include <map>
19#include <mutex>
20#include <set>
21#include <unordered_map>
22
23#include <Cocoa/Cocoa.h>
24#include <Metal/Metal.h>
25#include <QuartzCore/QuartzCore.h>
26
27@class CAMetalLayer;
28@class MTLCommandQueue;
29@class MTLRenderPipelineState;
30
31/* Metal Memory Manager Overview. */
32/*
33 * The Metal Backend Memory manager is designed to provide an interface
34 * for all other MTL_* modules where memory allocation is required.
35 *
36 * Different allocation strategies and data-structures are used depending
37 * on how the data is used by the backend. These aim to optimally handle
38 * system memory and abstract away any complexity from the MTL_* modules
39 * themselves.
40 *
41 * There are two primary allocation modes which can be used:
42 *
43 * ** MTLScratchBufferManager **
44 *
45 * Each MTLContext owns a ScratchBufferManager which is implemented
46 * as a pool of circular buffers, designed to handle temporary
47 * memory allocations which occur on a per-frame basis. The scratch
48 * buffers allow flushing of host memory to the GPU to be batched.
49 *
50 * Each frame, the next scratch buffer is reset, then later flushed upon
51 * command buffer submission.
52 *
53 * NOTE: This is allocated per-context due to allocations being tied
54 * to workload submissions and context-specific submissions.
55 *
56 * Examples of scratch buffer usage are:
57 * - Immediate-mode temporary vertex buffers.
58 * - Shader uniform data updates
59 * - Staging of data for resource copies, or, data reads/writes.
60 *
61 * Usage:
62 *
63 * MTLContext::get_scratch_buffer_manager() - to fetch active manager.
64 *
65 * MTLTemporaryBuffer scratch_buffer_allocate_range(size)
66 * MTLTemporaryBuffer scratch_buffer_allocate_range_aligned(size, align)
67 *
68 * ---------------------------------------------------------------------------------
69 * ** MTLBufferPool **
70 *
71 * For static and longer-lasting memory allocations, such as those for UBOs,
72 * Vertex buffers, index buffers, etc; We want an optimal abstraction for
73 * fetching a MTLBuffer of the desired size and resource options.
74 *
75 * Memory allocations can be expensive so the MTLBufferPool provides
76 * functionality to track usage of these buffers and once a buffer
77 * is no longer in use, it is returned to the buffer pool for use
78 * by another backend resource.
79 *
80 * The MTLBufferPool provides functionality for safe tracking of resources,
81 * as buffers freed on the host side must have their usage by the GPU tracked,
82 * to ensure they are not prematurely re-used before they have finished being
83 * used by the GPU.
84 *
85 * NOTE: The MTLBufferPool is a global construct which can be fetched from anywhere.
86 *
87 * Usage:
88 * MTLContext::get_global_memory_manager(); - static routine to fetch global memory manager.
89 *
90 * gpu::MTLBuffer *allocate(size, is_cpu_visible)
91 * gpu::MTLBuffer *allocate_aligned(size, alignment, is_cpu_visible)
92 * gpu::MTLBuffer *allocate_with_data(size, is_cpu_visible, data_ptr)
93 * gpu::MTLBuffer *allocate_aligned_with_data(size, alignment, is_cpu_visible, data_ptr)
94 */
95
96/* Debug memory statistics: Disabled by Macro rather than guarded for
97 * performance considerations. */
98#define MTL_DEBUG_MEMORY_STATISTICS 0
99
100namespace blender::gpu {
101
102/* Forward Declarations. */
103class MTLContext;
104class MTLCommandBufferManager;
105class MTLUniformBuf;
106class MTLStorageBuf;
107
108/* -------------------------------------------------------------------- */
111
112/* MTLBuffer allocation wrapper. */
114
115 public:
116 /* NOTE: ListBaseT API is not used due to custom destructor operation required to release
117 * Metal objective C buffer resource. */
119
120 private:
121 /* Metal resource. */
122 id<MTLBuffer> metal_buffer_;
123
124 /* Host-visible mapped-memory pointer. Behavior depends on buffer type:
125 * - Shared buffers: pointer represents base address of #MTLBuffer whose data
126 * access has shared access by both the CPU and GPU on
127 * Unified Memory Architectures (UMA).
128 * - Managed buffer: Host-side mapped buffer region for CPU (Host) access. Managed buffers
129 * must be manually flushed to transfer data to GPU-resident buffer.
130 * - Private buffer: Host access is invalid, `data` will be nullptr. */
131 void *data_;
132
133 /* Whether buffer is allocated from an external source. */
134 bool is_external_ = false;
135
136 /* Allocation info. */
137 MTLResourceOptions options_;
138 id<MTLDevice> device_;
139 uint64_t alignment_;
140 uint64_t size_;
141
142 /* Allocated size may be larger than actual size. */
143 uint64_t usage_size_;
144
145 /* Lifetime info - whether the current buffer is actively in use. A buffer
146 * should be in use after it has been allocated. De-allocating the buffer, and
147 * returning it to the free buffer pool will set in_use to false. Using a buffer
148 * while it is not in-use should not be allowed and result in an error. */
149 std::atomic<bool> in_use_;
150
151 public:
152 MTLBuffer(id<MTLDevice> device, uint64_t size, MTLResourceOptions options, uint alignment = 1);
153 MTLBuffer(id<MTLBuffer> external_buffer);
154 ~MTLBuffer();
155
156 /* Fetch information about backing MTLBuffer. */
158 void *get_host_ptr() const;
159 uint64_t get_size_used() const;
160 uint64_t get_size() const;
161
162 /* Flush data to GPU. */
163 void flush();
164 void flush_range(uint64_t offset, uint64_t length);
165 bool requires_flush();
166
167 /* Buffer usage tracking. */
168 void flag_in_use(bool used);
169 bool get_in_use();
170 void set_usage_size(uint64_t size_used);
171
172 /* Debug. */
173 void set_label(NSString *str);
174
175 /* Read properties. */
176 MTLResourceOptions get_resource_options();
178
179 /* Resource-local free: For buffers allocated via memory manager,
180 * this will call the context `free_buffer` method to return the buffer to the context memory
181 * pool.
182 *
183 * Otherwise, free will release the associated metal resource.
184 * As a note, calling the destructor will also destroy the buffer and associated metal
185 * resource. */
186 void free();
187
188 /* Safety check to ensure buffers are not used after free. */
189 void debug_ensure_used();
190
192};
193
194/* View into part of an MTLBuffer. */
197 void *data;
200 MTLResourceOptions options;
201
202 void flush();
203 bool requires_flush();
204};
205
206/* Circular scratch buffer allocations should be seen as temporary and only used within the
207 * lifetime of the frame. */
209
210/* Round-Robin Circular-buffer. */
213
214 private:
215 MTLContext &own_context_;
216
217 /* Wrapped MTLBuffer allocation handled. */
218 gpu::MTLBuffer *cbuffer_;
219 /* Allocated SSBO that serves as source for cbuffer. */
220 MTLStorageBuf *ssbo_source_ = nullptr;
221
222 /* Current offset where next allocation will begin. */
223 uint64_t current_offset_;
224
225 /* Whether the Circular Buffer can grow during re-allocation if
226 * the size is exceeded. */
227 bool can_resize_;
228
229 /* Usage information. */
230 uint64_t used_frame_index_;
231 uint64_t last_flush_base_offset_;
232
233 public:
234 MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow);
238 void flush();
239
240 /* Reset pointer back to start of circular buffer. */
241 void reset();
242};
243
244/* Wrapper struct used by Memory Manager to sort and compare gpu::MTLBuffer resources inside the
245 * memory pools. */
250
252 {
253 this->buffer = buf;
254 this->buffer_size = this->buffer->get_size();
255 this->insert_time = std::time(nullptr);
256 }
257
258 inline MTLBufferHandle(uint64_t compare_size)
259 {
260 this->buffer = nullptr;
261 this->buffer_size = compare_size;
262 this->insert_time = 0;
263 }
264};
265
267 bool operator()(const MTLBufferHandle &lhs, const MTLBufferHandle &rhs) const
268 {
269 return lhs.buffer_size < rhs.buffer_size;
270 }
271};
272
302 friend class MTLBufferPool;
303
304 private:
305 std::atomic<int> reference_count_;
306 std::atomic<bool> in_free_queue_;
307 std::atomic<bool> referenced_by_workload_;
308 std::recursive_mutex lock_;
309 /* Linked list of next MTLSafeFreeList chunk if current chunk is full. */
310 std::atomic<MTLSafeFreeList *> next_;
311 /* The list created after this one. It holds a reference released by this list, so that lists
312 * are released in order: a command buffer referencing a list also protects the buffers freed
313 * into later lists while it is encoded or executed. */
314 std::atomic<MTLSafeFreeList *> next_list_ = nullptr;
315
316 /* Lockless list. MAX_NUM_BUFFERS_ within a chunk based on considerations
317 * for performance and memory. Higher chunk counts are preferable for efficiently
318 * performing block operations such as copying several objects simultaneously.
319 *
320 * MIN_BUFFER_FLUSH_COUNT refers to the minimum count of buffers in the MTLSafeFreeList
321 * before buffers are returned to global memory pool. This is set at a point to reduce
322 * overhead of small pool flushes, while ensuring floating memory overhead is not excessive.
323 * MIN_FLUSH_SIZE does the same for lists of few, but large buffers. */
324 static const int MAX_NUM_BUFFERS_ = 8192;
325 static const int MIN_BUFFER_FLUSH_COUNT = 120;
326 static const uint64_t MIN_FLUSH_SIZE = 256 * 1024 * 1024;
327 std::atomic<int> current_list_index_;
328 std::atomic<uint64_t> freed_size_ = 0;
329 gpu::MTLBuffer *safe_free_pool_[MAX_NUM_BUFFERS_];
330
331 public:
333
334 /* Can be used from multiple threads. Performs insertion into Safe Free List with the least
335 * amount of threading synchronization. */
336 void insert_buffer(gpu::MTLBuffer *buffer);
337
338 /* Whether we need to start a new safe free list, or can carry on using the existing one. */
339 bool should_flush();
340
341 /* Increments command buffer reference count. */
342 void increment_reference();
343
344 /* Decrement and return of buffers to pool occur on MTLCommandBuffer completion callback. */
345 void decrement_reference();
346
348 {
349 in_free_queue_ = true;
350 if (current_list_index_ >= MTLSafeFreeList::MAX_NUM_BUFFERS_) {
351 MTLSafeFreeList *next_pool = next_.load();
352 if (next_pool) {
353 next_pool->flag_in_queue();
354 }
355 }
356 }
357
358 MEM_CXX_CLASS_ALLOC_FUNCS("MTLSafeFreeList");
359};
360
361/* MTLBuffer pools. */
362/* Allocating Metal buffers is expensive, so we cache all allocated buffers,
363 * and when requesting a new buffer, find one which fits the required dimensions
364 * from an existing pool of buffers.
365 *
366 * When freeing MTLBuffers, we insert them into the current MTLSafeFreeList, which defers
367 * release of the buffer until the associated command buffers have finished executing.
368 * This prevents a buffer from being re-used while it is still in-use by the GPU.
369 *
370 * * Once command buffers complete, MTLSafeFreeList's associated with the current
371 * command buffer submission are added to the `completed_safelist_queue_`.
372 *
373 * * At a set point in time, all MTLSafeFreeList's in `completed_safelist_queue_` have their
374 * MTLBuffers re-inserted into the Memory Manager's pools. */
376
377 private:
378#if MTL_DEBUG_MEMORY_STATISTICS == 1
379 /* Memory statistics. */
380 std::atomic<int64_t> total_allocation_bytes_;
381
382 /* Debug statistics. */
383 std::atomic<int> per_frame_allocation_count_;
384 std::atomic<int64_t> buffers_in_pool_;
385#endif
386
387 /* Metal resources. */
388 bool initialized_ = false;
389 id<MTLDevice> device_ = nil;
390
391 /* The buffer selection aims to pick a buffer which meets the minimum size requirements.
392 * To do this, we keep an ordered set of all available buffers. If the buffer is larger than the
393 * desired allocation size, we check it against `mtl_buffer_size_threshold_factor_`,
394 * which defines what % larger than the original allocation the buffer can be.
395 * - A higher value results in greater re-use of previously allocated buffers of similar sizes.
396 * - A lower value may result in more dynamic allocations, but minimized memory usage for a given
397 * scenario.
398 * The current value of 1.26 is calibrated for optimal performance and memory utilization. */
399 static constexpr float mtl_buffer_size_threshold_factor_ = 1.26;
400
401 /* Buffer pools using MTLResourceOptions as key for allocation type.
402 * Aliased as 'uint64_t' for map type compatibility.
403 * - A size-ordered list (MultiSet) of allocated buffers is kept per MTLResourceOptions
404 * permutation. This allows efficient lookup for buffers of a given requested size.
405 * - MTLBufferHandle wraps a gpu::MTLBuffer pointer to achieve easy size-based sorting
406 * via CompareMTLBuffer.
407 *
408 * NOTE: buffer_pool_lock_ guards against concurrent access to the memory allocator. This
409 * can occur during light baking or rendering operations. */
410 using MTLBufferPoolOrderedList = std::multiset<MTLBufferHandle, CompareMTLBuffer>;
411 using MTLBufferResourceOptions = uint64_t;
412
413 std::mutex buffer_pool_lock_;
415
416 /* Linked list to track all existing allocations. Prioritizing fast insert/deletion. */
417 gpu::MTLBuffer *allocations_list_base_;
418 uint allocations_list_size_;
419
420 /* Maintain a queue of all MTLSafeFreeList's that have been released
421 * by the GPU and are ready to have their buffers re-inserted into the
422 * MemoryManager pools.
423 * Access to this queue is made thread-safe through safelist_lock_. */
424 std::mutex safelist_lock_;
425 Vector<MTLSafeFreeList *> completed_safelist_queue_;
426
427 /* Current free list, associated with active MTLCommandBuffer submission. */
428 /* MTLBuffer::free() can be called from separate threads, due to usage within animation
429 * system/worker threads. */
430 std::atomic<MTLSafeFreeList *> current_free_list_;
431 std::atomic<int64_t> allocations_in_pool_;
432
433 /* Previous list, to be released after one full frame. */
434 MTLSafeFreeList *prev_free_buffer_list_ = nullptr;
435
436 public:
437 void init(id<MTLDevice> device);
439
440 gpu::MTLBuffer *allocate(uint64_t size, bool cpu_visible);
441 gpu::MTLBuffer *allocate_aligned(uint64_t size, uint alignment, bool cpu_visible);
442 gpu::MTLBuffer *allocate_with_data(uint64_t size, bool cpu_visible, const void *data = nullptr);
444 uint alignment,
445 bool cpu_visible,
446 const void *data = nullptr);
447 bool free_buffer(gpu::MTLBuffer *buffer);
448
449 /* Flush MTLSafeFreeList buffers, for completed lists in `completed_safelist_queue_`,
450 * back to memory pools. */
451 void update_memory_pools();
453
454 /* Access and control over active MTLSafeFreeList. The list can be released by other threads
455 * unless `safelist_lock_` is held, use #acquire_current_safe_list to keep using it. */
457 /* Reference the current list, released with #MTLSafeFreeList::decrement_reference. */
459 void begin_new_safe_list(bool only_if_should_flush = false);
460
461 /* Add a completed MTLSafeFreeList to completed_safelist_queue_. */
463
464 private:
465 void ensure_buffer_pool(MTLResourceOptions options);
466 void insert_buffer_into_pool(MTLResourceOptions options, gpu::MTLBuffer *buffer);
467 void free();
468
469 /* Allocations list. */
470 void allocations_list_insert(gpu::MTLBuffer *buffer);
471 void allocations_list_delete(gpu::MTLBuffer *buffer);
472 void allocations_list_delete_all();
473};
474
475/* Scratch buffers are circular-buffers used for temporary data within the current frame.
476 * In order to preserve integrity of contents when having multiple-frames-in-flight,
477 * we cycle through a collection of scratch buffers which are reset upon next use.
478 *
479 * Below are a series of properties, declared to manage scratch buffers. If a scratch buffer
480 * overflows, then the original buffer will be flushed and submitted, with retained references
481 * by usage within the command buffer, and a new buffer will be created.
482 * - The new buffer will grow in size to account for increased demand in temporary memory.
483 */
485
486 private:
487 /* Maximum number of scratch buffers to allocate. This should be the maximum number of
488 * simultaneous frames in flight. */
489 static constexpr uint mtl_max_scratch_buffers_ = MTL_NUM_SAFE_FRAMES;
490
491 public:
492 /* Maximum size of single scratch buffer allocation. When re-sizing, this is the maximum size the
493 * newly allocated buffers will grow to. Larger allocations are possible if
494 * `MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION` is enabled, but these will instead allocate new
495 * buffers from the memory pools on the fly. */
496 static constexpr uint mtl_scratch_buffer_max_size_ = 128 * 1024 * 1024;
497
498 /* Initial size of circular scratch buffers prior to growth. */
499 static constexpr uint mtl_scratch_buffer_initial_size_ = 16 * 1024 * 1024;
500
501 private:
502 /* Parent MTLContext. */
503 MTLContext &context_;
504 bool initialised_ = false;
505
506 /* Scratch buffer currently in-use. */
507 uint current_scratch_buffer_ = 0;
508
509 /* Scratch buffer pool. */
510 MTLCircularBuffer *scratch_buffers_[mtl_max_scratch_buffers_];
511
512 public:
513 MTLScratchBufferManager(MTLContext &context) : context_(context) {};
515
516 /* Explicit initialization and freeing of resources.
517 * Initialization must occur after device creation. */
518 void init();
519 void free();
520
521 /* Allocation functions for creating temporary allocations from active circular buffer. */
524
525 /* Ensure a new scratch buffer is started if we move onto a new frame.
526 * Called when a new command buffer begins. */
528
529 /* Flush memory for active scratch buffer to GPU.
530 * This call will perform a partial flush of the buffer starting from
531 * the last offset the data was flushed from, to the current offset. */
533
534 /* Bind the whole scratch buffer as a SSBO resource. */
535 void bind_as_ssbo(int slot);
536 void unbind_as_ssbo();
537
539};
540
542
543} // namespace blender::gpu
void push_completed_safe_list(MTLSafeFreeList *list)
void init(id< MTLDevice > device)
Definition mtl_memory.mm:40
gpu::MTLBuffer * allocate_with_data(uint64_t size, bool cpu_visible, const void *data=nullptr)
MTLSafeFreeList * acquire_current_safe_list()
MTLSafeFreeList * get_current_safe_list()
gpu::MTLBuffer * allocate(uint64_t size, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned(uint64_t size, uint alignment, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned_with_data(uint64_t size, uint alignment, bool cpu_visible, const void *data=nullptr)
bool free_buffer(gpu::MTLBuffer *buffer)
void begin_new_safe_list(bool only_if_should_flush=false)
void flag_in_use(bool used)
uint64_t get_size() const
void set_usage_size(uint64_t size_used)
gpu::MTLBuffer * next
uint64_t get_size_used() const
void * get_host_ptr() const
void flush_range(uint64_t offset, uint64_t length)
MTLBuffer(id< MTLDevice > device, uint64_t size, MTLResourceOptions options, uint alignment=1)
gpu::MTLBuffer * prev
void set_label(NSString *str)
MTLResourceOptions get_resource_options()
id< MTLBuffer > get_metal_buffer() const
MEM_CXX_CLASS_ALLOC_FUNCS("MTLBuffer")
MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow)
MTLTemporaryBuffer allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer allocate_range(uint64_t alloc_size)
MEM_CXX_CLASS_ALLOC_FUNCS("MTLSafeFreeList")
void insert_buffer(gpu::MTLBuffer *buffer)
static constexpr uint mtl_scratch_buffer_max_size_
static constexpr uint mtl_scratch_buffer_initial_size_
MTLScratchBufferManager(MTLContext &context)
MTLTemporaryBuffer scratch_buffer_allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer scratch_buffer_allocate_range(uint64_t alloc_size)
CCL_NAMESPACE_BEGIN struct Options options
#define str(s)
float length(VecOp< float, D >) RET
#define MTL_NUM_SAFE_FRAMES
Definition mtl_common.hh:19
void free(Subdiv *subdiv)
Definition subdiv.cc:192
MTLBufferRange MTLTemporaryBuffer
unsigned __int64 uint64_t
Definition BKE_text.h:91
unsigned int uint
Definition BKE_text.h:55
BMesh const char void * data
bool operator()(const MTLBufferHandle &lhs, const MTLBufferHandle &rhs) const
MTLBufferHandle(gpu::MTLBuffer *buf)
MTLBufferHandle(uint64_t compare_size)
MTLResourceOptions options