Blender V5.3
mtl_storage_buffer.mm
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2022 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include "BLI_string.hh"
10#include "BLI_time.hh"
11
12#include "GPU_state.hh"
13#include "gpu_backend.hh"
15
16#include "mtl_backend.hh"
17#include "mtl_context.hh"
18#include "mtl_debug.hh"
19#include "mtl_index_buffer.hh"
20#include "mtl_memory.hh"
21#include "mtl_storage_buffer.hh"
22#include "mtl_uniform_buffer.hh"
23#include "mtl_vertex_buffer.hh"
24#include <cstdio>
25
26namespace blender::gpu {
27
28static CLG_LogRef LOG = {"gpu.metal"};
29
30/* -------------------------------------------------------------------- */
33
34MTLStorageBuf::MTLStorageBuf(size_t size) : StorageBuf(size, "Immediate")
35{
36 usage_ = GPU_USAGE_STREAM;
37 storage_source_ = MTL_STORAGE_BUF_TYPE_DEFAULT;
38 metal_buffer_ = MTLContext::get_global_memory_manager()->allocate_aligned(size, 256, true);
39}
40
41MTLStorageBuf::MTLStorageBuf(size_t size, GPUUsageType usage, const char *name)
42 : StorageBuf(size, name)
43{
44 usage_ = usage;
45 /* Do not create SSBO MTL buffer here to allow allocation from any thread. */
46 storage_source_ = MTL_STORAGE_BUF_TYPE_DEFAULT;
47 metal_buffer_ = nullptr;
48}
49
51 : StorageBuf(size, "UniformBuffer_as_SSBO")
52{
53 usage_ = GPU_USAGE_DYNAMIC;
54 storage_source_ = MTL_STORAGE_BUF_TYPE_UNIFORMBUF;
55 uniform_buffer_ = uniform_buf;
56 BLI_assert(uniform_buffer_ != nullptr);
57}
58
60 : StorageBuf(size, "VertexBuffer_as_SSBO")
61{
62 usage_ = GPU_USAGE_DYNAMIC;
63 storage_source_ = MTL_STORAGE_BUF_TYPE_VERTBUF;
64 vertex_buffer_ = vert_buf;
65 BLI_assert(vertex_buffer_ != nullptr);
66}
67
69 : StorageBuf(size, "IndexBuffer_as_SSBO")
70{
71 usage_ = GPU_USAGE_DYNAMIC;
72 storage_source_ = MTL_STORAGE_BUF_TYPE_INDEXBUF;
73 index_buffer_ = index_buf;
74 BLI_assert(index_buffer_ != nullptr);
75}
76
78 : StorageBuf(size, "Texture_as_SSBO")
79{
80 usage_ = GPU_USAGE_DYNAMIC;
81 storage_source_ = MTL_STORAGE_BUF_TYPE_TEXTURE;
82 texture_ = texture;
83 BLI_assert(texture_ != nullptr);
84}
85
87{
88 if (storage_source_ == MTL_STORAGE_BUF_TYPE_DEFAULT) {
89 if (metal_buffer_ != nullptr) {
90 metal_buffer_->free();
91 metal_buffer_ = nullptr;
92 }
93 has_data_ = false;
94 }
95
96 if (gpu_write_fence_ != nil) {
97 [gpu_write_fence_ release];
98 gpu_write_fence_ = nil;
99 }
100
101 /* Ensure SSBO is not bound to active CTX.
102 * SSBO bindings are reset upon Context-switch so we do not need
103 * to check deactivated context's. */
105 if (ctx) {
107 if (slot.bound && slot.ssbo == this) {
108 slot.bound = false;
109 slot.ssbo = nullptr;
110 }
111 }
112 }
113}
114
116
117/* -------------------------------------------------------------------- */
120
122{
123 /* We only need to initialize the storage buffer for default buffer types. */
124 if (storage_source_ != MTL_STORAGE_BUF_TYPE_DEFAULT) {
125 return;
126 }
127 BLI_assert(this);
129
130 /* Allocate MTL buffer */
132 BLI_assert(ctx);
133 BLI_assert(ctx->device);
135
137 size_in_bytes_, (usage_ == GPU_USAGE_DEVICE_ONLY) ? false : true);
138
139#ifndef NDEBUG
140 static std::atomic<int> global_counter = 0;
141 int index = global_counter.fetch_add(1);
142 metal_buffer_->set_label([NSString stringWithFormat:@"SSBO %i %s", index, name_]);
143#endif
144 BLI_assert(metal_buffer_ != nullptr);
145 BLI_assert(metal_buffer_->get_metal_buffer() != nil);
146
147 has_data_ = false;
148}
149
151{
152 /* We only need to initialize the storage buffer for default buffer types. */
153 if (storage_source_ != MTL_STORAGE_BUF_TYPE_DEFAULT) {
154 return;
155 }
156
157 /* For device-only Storage buffers, update private resource via staging buffer in command
158 * stream. */
159 bool device_only = (usage_ == GPU_USAGE_DEVICE_ONLY);
160 bool do_upload_data = (data != nullptr);
161
162 /* If host-visible, upload data to new buffer, as previous data may still be in-use by executing
163 * GPU commands. */
164 if (!device_only && do_upload_data) {
165 if (metal_buffer_ != nullptr) {
166 metal_buffer_->free();
167 metal_buffer_ = nullptr;
168 }
169 }
170
171 /* Ensure buffer has been allocated. */
172 if (metal_buffer_ == nullptr) {
173 init();
174 }
175
176 BLI_assert(do_upload_data);
177 if (do_upload_data) {
178 if (device_only) {
179
180 /* Fetch active context. */
182 BLI_assert(ctx);
183
184 /* Prepare staging buffer. */
186 size_in_bytes_, true);
187 memcpy(staging_buf->get_host_ptr(), data, size_in_bytes_);
188 staging_buf->flush_range(0, size_in_bytes_);
189 id<MTLBuffer> staging_buf_mtl = staging_buf->get_metal_buffer();
190 BLI_assert(staging_buf_mtl != nil);
191
192 /* Ensure destination buffer. */
193 id<MTLBuffer> dst_buf = this->metal_buffer_->get_metal_buffer();
194 BLI_assert(dst_buf != nil);
195
196 id<MTLBlitCommandEncoder> blit_encoder =
198 [blit_encoder copyFromBuffer:staging_buf_mtl
199 sourceOffset:0
200 toBuffer:dst_buf
201 destinationOffset:0
202 size:size_in_bytes_];
203 staging_buf->free();
204 }
205 else {
206 /* Upload data. */
207 BLI_assert(data != nullptr);
208 BLI_assert(!(metal_buffer_->get_resource_options() & MTLResourceStorageModePrivate));
210 BLI_assert(size_in_bytes_ <= [metal_buffer_->get_metal_buffer() length]);
211 memcpy(metal_buffer_->get_host_ptr(), data, size_in_bytes_);
212 metal_buffer_->flush_range(0, size_in_bytes_);
213 }
214 has_data_ = true;
215 }
216}
217
219
220/* -------------------------------------------------------------------- */
223
225{
226 if (slot >= MTL_MAX_BUFFER_BINDINGS) {
228 "Error: Trying to bind \"%s\" ssbo to slot %d which is above the reported limit of "
229 "%d.",
230 name_,
231 slot,
233 BLI_assert(false);
234 return;
235 }
236
237 if (metal_buffer_ == nullptr) {
238 this->init();
239 }
240
241 if (data_ != nullptr) {
242 this->update(data_);
243 MEM_SAFE_DELETE_VOID(data_);
244 }
245
246 /* Bind current UBO to active context. */
248 BLI_assert(ctx);
249
250 MTLStorageBufferBinding &ctx_ssbo_bind_slot = ctx->pipeline_state.ssbo_bindings[slot];
251 ctx_ssbo_bind_slot.ssbo = this;
252 ctx_ssbo_bind_slot.bound = true;
253
254 bind_slot_ = slot;
255 bound_ctx_ = ctx;
256}
257
259{
260 /* Unbind in debug mode to validate missing binds.
261 * Otherwise, only perform a full unbind upon destruction
262 * to ensure no lingering references. */
263#ifndef NDEBUG
264 if (true)
265#else
266 if (G.debug & G_DEBUG_GPU)
267#endif
268 {
269 if (bound_ctx_ != nullptr && bind_slot_ > -1) {
270 MTLStorageBufferBinding &ctx_ssbo_bind_slot =
271 bound_ctx_->pipeline_state.ssbo_bindings[bind_slot_];
272 if (ctx_ssbo_bind_slot.bound && ctx_ssbo_bind_slot.ssbo == this) {
273 ctx_ssbo_bind_slot.bound = false;
274 ctx_ssbo_bind_slot.ssbo = nullptr;
275 }
276 }
277 }
278
279 /* Reset bind index. */
280 bind_slot_ = -1;
281 bound_ctx_ = nullptr;
282}
283
285{
286 /* Fetch active context. */
288 BLI_assert_msg(ctx, "Clears should always be performed while a valid context exists.");
289
290 if (metal_buffer_ == nullptr) {
291 this->init();
292 }
293
294 if (ctx) {
295 /* If all 4 bytes within clear value are equal, use the builtin fast-path for clearing. */
296 uint clear_byte = clear_value & 0xFF;
297 bool clear_value_bytes_equal = (clear_byte == ((clear_value >> 8) & 0xFF)) &&
298 (clear_byte == ((clear_value >> 16) & 0xFF)) &&
299 (clear_byte == ((clear_value >> 24) & 0xFF));
300 if (clear_value_bytes_equal) {
301 id<MTLBlitCommandEncoder> blit_encoder =
303 [blit_encoder fillBuffer:metal_buffer_->get_metal_buffer()
304 range:NSMakeRange(0, size_in_bytes_)
305 value:clear_byte];
306 }
307 else {
308 /* We need a special compute routine to update 32 bit values efficiently. */
310 id<MTLComputeCommandEncoder> compute_encoder =
312
314 cs.bind_pso(pso);
315 cs.bind_compute_bytes(&clear_value, sizeof(uint32_t), 0);
316 cs.bind_compute_buffer(metal_buffer_->get_metal_buffer(), 0, 1);
317 [compute_encoder dispatchThreads:MTLSizeMake(size_in_bytes_ / sizeof(uint32_t), 1, 1)
318 threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
319 }
320 }
321}
322
323void MTLStorageBuf::copy_sub(VertBuf *src_, uint dst_offset, uint src_offset, uint copy_size)
324{
325 MTLVertBuf *src = static_cast<MTLVertBuf *>(src_);
326 MTLStorageBuf *dst = this;
327
328 if (dst->metal_buffer_ == nullptr) {
329 dst->init();
330 }
331 if (copy_size == 0) {
332 return;
333 }
334 if (src->vbo_ == nullptr) {
335 src->bind();
336 }
337
338 /* Fetch active context. */
340 BLI_assert(ctx);
341
342 /* Fetch Metal buffers. */
343 id<MTLBuffer> src_buf = src->vbo_->get_metal_buffer();
344 id<MTLBuffer> dst_buf = dst->metal_buffer_->get_metal_buffer();
345 BLI_assert(src_buf != nil);
346 BLI_assert(dst_buf != nil);
347
349 [blit_encoder copyFromBuffer:src_buf
350 sourceOffset:src_offset
351 toBuffer:dst_buf
352 destinationOffset:dst_offset
353 size:copy_size];
354}
355
357{
358 bool device_only = (usage_ == GPU_USAGE_DEVICE_ONLY);
359 BLI_assert_msg(!device_only,
360 "Storage buffers with usage GPU_USAGE_DEVICE_ONLY cannot have their data "
361 "synchronized to the host.");
362 if (device_only) {
363 return;
364 }
365
367 BLI_assert(ctx);
368
369 if (metal_buffer_ == nullptr) {
370 this->init();
371 }
372
373 /* Unified memory: GPU writes are host-visible after command-buffer completion, so the
374 * sync blit and signal event are unnecessary; read() falls back to GPU_finish() when
375 * gpu_write_fence_ is nil. Skipping the signal event also avoids a crash in AGXMetalG17X
376 * (M5 family) where encodeSignalEvent: aborts inside the driver. */
377 id<MTLBuffer> storage_buf_mtl = this->metal_buffer_->get_metal_buffer();
378 if (storage_buf_mtl.storageMode != MTLStorageModeManaged) {
379 return;
380 }
381
382 if (gpu_write_fence_ == nil) {
383 gpu_write_fence_ = [ctx->device newSharedEvent];
384 }
385
387 [blit_encoder synchronizeResource:storage_buf_mtl];
388
389 ctx->main_command_buffer.encode_signal_event(gpu_write_fence_, ++host_read_signal_value_);
390 ctx->flush();
391}
392
394{
395 if (data == nullptr) {
396 return;
397 }
398
399 if (metal_buffer_ == nullptr) {
400 this->init();
401 }
402
403 /* Device-only storage buffers cannot be read directly and require staging.
404 * This path should only be used for unit testing. */
405 bool device_only = (usage_ == GPU_USAGE_DEVICE_ONLY);
406 if (device_only) {
408 /* Fetch active context. */
410 BLI_assert(ctx);
411
412 /* Prepare staging buffer. */
414 true);
415 id<MTLBuffer> staging_buf_mtl = staging_buf->get_metal_buffer();
416 BLI_assert(staging_buf_mtl != nil);
417
418 /* Ensure destination buffer. */
419 id<MTLBuffer> storage_buf_mtl = this->metal_buffer_->get_metal_buffer();
420 BLI_assert(storage_buf_mtl != nil);
421
423 [blit_encoder copyFromBuffer:storage_buf_mtl
424 sourceOffset:0
425 toBuffer:staging_buf_mtl
426 destinationOffset:0
427 size:size_in_bytes_];
428 if (staging_buf_mtl.storageMode == MTLStorageModeManaged) {
429 [blit_encoder synchronizeResource:staging_buf_mtl];
430 }
431
432 /* Device-only reads will always stall the GPU pipe. */
433 GPU_finish();
435 "Device-only storage buffer being read. This will stall the GPU pipeline. Ensure this "
436 "path is only used in testing.");
437
438 /* Read contents back to data. */
439 memcpy(data, staging_buf->get_host_ptr(), size_in_bytes_);
440 staging_buf->free();
441 }
442 else {
444 /* If we have a synchronization event from a prior memory sync, ensure memory is fully synced.
445 * Otherwise, assume read is synchronous and stall until in-flight work is complete. */
446 if (gpu_write_fence_ != nil) {
447 /* Ensure the GPU updates are visible to the host before reading. */
448 while (gpu_write_fence_.signaledValue < host_read_signal_value_) {
450 }
451 }
452 else {
453 /* In the case of unified memory. Wait for all pending operation. */
454 GPU_finish();
455 }
456
457 /* Managed buffers need to be explicitly flushed back to host. */
458 if (metal_buffer_->get_resource_options() & MTLResourceStorageModeManaged) {
459 /* Fetch active context. */
461 BLI_assert(ctx);
462
463 /* Ensure GPU updates are flushed back to CPU. */
464 id<MTLBlitCommandEncoder> blit_encoder =
466 [blit_encoder synchronizeResource:metal_buffer_->get_metal_buffer()];
467
468 /* Wait for the blit to finish. */
469 GPU_finish();
470 }
471
472 /* Read data. NOTE: Unless explicitly synchronized with GPU work, results may not be ready. */
473 memcpy(data, metal_buffer_->get_host_ptr(), size_in_bytes_);
474 }
475}
476
478{
479
480 gpu::MTLBuffer *source_buffer = nullptr;
481 switch (storage_source_) {
482 /* Default SSBO buffer comes from own allocation. */
483 case MTL_STORAGE_BUF_TYPE_DEFAULT: {
484 /* NOTE: We should always ensure that the data is primed prior to requiring fetching of the
485 * buffer. If get_metal_buffer is called during a resource bind phase, invoking a blit
486 * command encoder to upload data would override the active encoder state being prepared.
487 * Resource generation and data upload should happen earlier as a resource is bound. */
488 BLI_assert_msg(metal_buffer_ != nullptr,
489 "Storage Buffer backing resource does not yet exist. Ensure the resource is "
490 "bound with data before the calling code requires its underlying MTLBuffer.");
492 data_ == nullptr,
493 "Storage Buffer backing resource data has not yet been uploaded. Ensure the resource is "
494 "bound with data before the calling code requires its underlying MTLBuffer.");
495 source_buffer = metal_buffer_;
496 } break;
497 /* SSBO buffer comes from Uniform Buffer. */
498 case MTL_STORAGE_BUF_TYPE_UNIFORMBUF: {
499 source_buffer = uniform_buffer_->metal_buffer_;
500 } break;
501 /* SSBO buffer comes from Vertex Buffer. */
502 case MTL_STORAGE_BUF_TYPE_VERTBUF: {
503 source_buffer = vertex_buffer_->vbo_;
504 } break;
505 /* SSBO buffer comes from Index Buffer. */
506 case MTL_STORAGE_BUF_TYPE_INDEXBUF: {
507 source_buffer = index_buffer_->ibo_;
508 } break;
509 /* SSBO buffer comes from Texture. */
510 case MTL_STORAGE_BUF_TYPE_TEXTURE: {
511 BLI_assert(texture_);
512 /* Fetch metal texture to ensure it has been initialized. */
513 id<MTLTexture> tex = texture_->get_metal_handle_base();
514 BLI_assert(tex != nil);
516 source_buffer = texture_->backing_buffer_;
517 }
518 }
519
520 /* Return Metal allocation handle and flag as used. */
521 BLI_assert(source_buffer != nullptr);
522 source_buffer->debug_ensure_used();
523 return source_buffer->get_metal_buffer();
524}
525
527{
528 BLI_assert(this);
529 return size_in_bytes_;
530}
531
533
534} // namespace blender::gpu
Platform independent time functions.
#define CLOG_ERROR(clg_ref,...)
Definition CLG_log.h:188
#define UNUSED_VARS_NDEBUG(...)
#define BLI_assert_msg(a, msg)
gpu::MTLBuffer * allocate(uint64_t size, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned(uint64_t size, uint alignment, bool cpu_visible)
void * get_host_ptr() const
void flush_range(uint64_t offset, uint64_t length)
id< MTLBuffer > get_metal_buffer() const
void encode_signal_event(id< MTLEvent > event, uint64_t value)
id< MTLComputeCommandEncoder > ensure_begin_compute_encoder()
id< MTLBlitCommandEncoder > ensure_begin_blit_encoder()
void bind_compute_bytes(const void *bytes, uint64_t length, uint index)
void bind_compute_buffer(id< MTLBuffer > buffer, uint64_t buffer_offset, uint index)
void bind_pso(id< MTLComputePipelineState > pso)
id< MTLComputePipelineState > get_buffer_clear_pso()
MTLContextComputeUtils & get_compute_utils()
static MTLContext * get()
MTLContextGlobalShaderPipelineState pipeline_state
MTLCommandBufferManager main_command_buffer
static MTLBufferPool * get_global_memory_manager()
void read(void *data) override
void clear(uint32_t clear_value) override
void copy_sub(VertBuf *src, uint dst_offset, uint src_offset, uint copy_size) override
void update(const void *data) override
MTLStorageBuf(size_t size, GPUUsageType usage, const char *name)
void bind(int slot) override
StorageBuf(size_t size, const char *name)
float length(VecOp< float, D >) RET
TEX_TEMPLATE DataVec texture(T, FltCoord, float=0.0f) RET
#define MTL_MAX_BUFFER_BINDINGS
#define MTL_LOG_WARNING(info,...)
Definition mtl_debug.hh:39
static CLG_LogRef LOG
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
unsigned int uint
Definition BKE_text.h:55
void BLI_time_sleep_ms(int ms)
Definition time.cc:91
Global G
Definition blender.cc:57
unsigned int uint32_t
Definition BKE_text.h:81
BMesh const char void * data
void GPU_finish()
Definition gpu_state.cc:289
std::array< MTLStorageBufferBinding, MTL_MAX_SSBO > ssbo_bindings