Blender V5.3
cycles/device/memory.h
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#pragma once
6
7/* Device Memory
8 *
9 * Data types for allocating, copying and freeing device memory. */
10
11#include "util/array.h"
12#include "util/half.h"
13#include "util/string.h"
14#include "util/types.h"
15#include "util/types_image.h"
16
18
19class Device;
20class GPUDevice;
21class CUDADevice;
22class OptiXDevice;
23class HIPDevice;
24class HIPRTDevice;
25class MetalDevice;
26class OneapiDevice;
27
35
37 /* Never map to host memory when the device runs out of memory, where using
38 * GPU memory is essential for performance. Scene data will then be moved to
39 * the host instead. */
41 /* Allocate a copy on every device instead of distributing across peer devices,
42 * for memory that devices write to during rendering. */
44};
45
46/* Supported Data Types */
47
59
60static constexpr size_t datatype_size(DataType datatype)
61{
62 switch (datatype) {
63 case TYPE_UNKNOWN:
64 return 1;
65 case TYPE_UCHAR:
66 return sizeof(uchar);
67 case TYPE_FLOAT:
68 return sizeof(float);
69 case TYPE_UINT:
70 return sizeof(uint);
71 case TYPE_UINT16:
72 return sizeof(uint16_t);
73 case TYPE_INT:
74 return sizeof(int);
75 case TYPE_INT8:
76 return sizeof(int8_t);
77 case TYPE_HALF:
78 return sizeof(half);
79 case TYPE_UINT64:
80 return sizeof(uint64_t);
81 default:
82 return 0;
83 }
84}
85
86/* Traits for data types */
87
88template<typename T> struct device_type_traits {
90 static const size_t num_elements = sizeof(T);
91};
92
93template<> struct device_type_traits<uchar> {
95 static const size_t num_elements = 1;
96 static_assert(sizeof(uchar) == num_elements * datatype_size(data_type));
97};
98
99template<> struct device_type_traits<uchar2> {
101 static const size_t num_elements = 2;
102 static_assert(sizeof(uchar2) == num_elements * datatype_size(data_type));
103};
104
105template<> struct device_type_traits<uchar3> {
107 static const size_t num_elements = 3;
108 static_assert(sizeof(uchar3) == num_elements * datatype_size(data_type));
109};
110
111template<> struct device_type_traits<uchar4> {
113 static const size_t num_elements = 4;
114 static_assert(sizeof(uchar4) == num_elements * datatype_size(data_type));
115};
116
117template<> struct device_type_traits<uint> {
119 static const size_t num_elements = 1;
120 static_assert(sizeof(uint) == num_elements * datatype_size(data_type));
121};
122
123template<> struct device_type_traits<uint2> {
125 static const size_t num_elements = 2;
126 static_assert(sizeof(uint2) == num_elements * datatype_size(data_type));
127};
128
129template<> struct device_type_traits<uint3> {
130 /* uint3 has different size depending on the device, can't use it for interchanging
131 * memory between CPU and GPU.
132 *
133 * Leave body empty to trigger a compile error if used. */
134};
135
136template<> struct device_type_traits<uint4> {
138 static const size_t num_elements = 4;
139 static_assert(sizeof(uint4) == num_elements * datatype_size(data_type));
140};
141
142template<> struct device_type_traits<int> {
144 static const size_t num_elements = 1;
145 static_assert(sizeof(int) == num_elements * datatype_size(data_type));
146};
147
148template<> struct device_type_traits<int2> {
150 static const size_t num_elements = 2;
151 static_assert(sizeof(int2) == num_elements * datatype_size(data_type));
152};
153
154template<> struct device_type_traits<int3> {
155 /* int3 has different size depending on the device, can't use it for interchanging
156 * memory between CPU and GPU.
157 *
158 * Leave body empty to trigger a compile error if used. */
159};
160
161template<> struct device_type_traits<int4> {
163 static const size_t num_elements = 4;
164 static_assert(sizeof(int4) == num_elements * datatype_size(data_type));
165};
166
167template<> struct device_type_traits<float> {
169 static const size_t num_elements = 1;
170 static_assert(sizeof(float) == num_elements * datatype_size(data_type));
171};
172
173template<> struct device_type_traits<float2> {
175 static const size_t num_elements = 2;
176 static_assert(sizeof(float2) == num_elements * datatype_size(data_type));
177};
178
179template<> struct device_type_traits<float3> {
180 /* float3 has different size depending on the device, can't use it for interchanging
181 * memory between CPU and GPU.
182 *
183 * Leave body empty to trigger a compile error if used. */
184};
185
188 static const size_t num_elements = 3;
189 static_assert(sizeof(packed_float3) == num_elements * datatype_size(data_type));
190};
191
192template<> struct device_type_traits<float4> {
194 static const size_t num_elements = 4;
195 static_assert(sizeof(float4) == num_elements * datatype_size(data_type));
196};
197
198template<> struct device_type_traits<half> {
200 static const size_t num_elements = 1;
201 static_assert(sizeof(half) == num_elements * datatype_size(data_type));
202};
203
204template<> struct device_type_traits<ushort4> {
206 static const size_t num_elements = 4;
207 static_assert(sizeof(ushort4) == num_elements * datatype_size(data_type));
208};
209
210template<> struct device_type_traits<uint16_t> {
212 static const size_t num_elements = 1;
213 static_assert(sizeof(uint16_t) == num_elements * datatype_size(data_type));
214};
215
218 static const size_t num_elements = 3;
219 static_assert(sizeof(packed_half3) == num_elements * datatype_size(data_type));
220};
221
222template<> struct device_type_traits<half4> {
224 static const size_t num_elements = 4;
225 static_assert(sizeof(half4) == num_elements * datatype_size(data_type));
226};
227
228template<> struct device_type_traits<uint64_t> {
230 static const size_t num_elements = 1;
231 static_assert(sizeof(uint64_t) == num_elements * datatype_size(data_type));
232};
233
234template<> struct device_type_traits<Quaternion> {
236 static const size_t num_elements = 4;
237 static_assert(sizeof(Quaternion) == num_elements * datatype_size(data_type));
238};
239
244
245/* Device Memory
246 *
247 * Base class for all device memory. This should not be allocated directly,
248 * instead the appropriate subclass can be used. */
249
251 public:
252 size_t memory_size()
253 {
255 }
256 size_t memory_elements_size(const int elements)
257 {
258 return elements * data_elements * datatype_size(data_type);
259 }
260
261 const char *global_name() const;
262 virtual string log_name() const;
263
264 /* Data information. */
267 size_t data_size;
272
273 /* Pointers. */
278 /* reference counter for shared_pointer */
280 bool move_to_host = false;
281
282 /* MemoryFlag. */
283 uint32_t flags;
284
285 virtual ~device_memory();
286
287 void swap_device(Device *new_device, const size_t new_device_size, device_ptr new_device_ptr);
288 void restore_device();
289
290 bool is_resident(Device *sub_device) const;
291 bool is_shared(Device *sub_device) const;
292
293 /* No copying and allowed.
294 *
295 * This is because device implementation might need to register device memory in an allocation
296 * map of some sort and use pointer as a key to identify blocks. Moving data from one place to
297 * another bypassing device allocation routines will make those maps hard to maintain. */
298 device_memory(const device_memory &) = delete;
299 device_memory(device_memory &&other) noexcept = delete;
302
303 protected:
304 friend class Device;
305 friend class GPUDevice;
306 friend class CUDADevice;
307 friend class OptiXDevice;
308 friend class HIPDevice;
309 friend class HIPRTDevice;
310 friend class MetalDevice;
311 friend class OneapiDevice;
312
313 /* Only create through subclasses. */
314 device_memory(Device *device, const char *name, MemoryType type, uint32_t flags = 0);
315
316 /* Host allocation on the device. All host_pointer memory should be
317 * allocated with these functions, for devices that support using
318 * the same pointer for host and device. */
319 void *host_alloc(const size_t size);
320
321 /* Device memory allocation and copying. */
322 void device_alloc();
323 void device_copy_to();
324 void device_move_to_host();
325 void device_copy_from(const size_t y, const size_t w, size_t h, const size_t elem);
326 void device_copy_merged_bitmap_from(const size_t y, const size_t w, size_t h);
327 void device_zero();
328
329 /* Memory can only be freed on host and device together. */
331 /* Free only the host buffer, leaving any device allocation intact. */
332 void host_only_free();
333
334 bool device_is_cpu();
335
336 const char *name_;
342};
343
344/* Device Only Memory
345 *
346 * Working memory only needed by the device, with no corresponding allocation
347 * on the host. Only used internally in the device implementations. */
348
349template<typename T> class device_only_memory : public device_memory {
350 public:
351 device_only_memory(Device *device, const char *name, bool allow_host_memory_fallback = false)
352 : device_memory(device, name, allow_host_memory_fallback ? MEM_READ_WRITE : MEM_DEVICE_ONLY)
353 {
356 }
357
358 device_only_memory(device_only_memory &&other) noexcept : device_memory(std::move(other)) {}
359
361 {
362 free();
363 }
364
365 void alloc_to_device(const size_t num, bool shrink_to_fit = true)
366 {
367 size_t new_size = num;
368 bool reallocate;
369
370 if (shrink_to_fit) {
371 reallocate = (data_size != new_size);
372 }
373 else {
374 reallocate = (data_size < new_size);
375 }
376
377 if (reallocate) {
379 data_size = new_size;
380 device_alloc();
381 }
382 }
383
384 void free()
385 {
387 data_size = 0;
388 }
389
391 {
392 device_zero();
393 }
394};
395
396/* Device Vector
397 *
398 * Data vector to exchange data between host and device. Memory will be
399 * allocated on the host first with alloc() and resize, and then filled
400 * in and copied to the device with copy_to_device(). Or alternatively
401 * allocated and set to zero on the device with zero_to_device().
402 *
403 * When using memory type MEM_GLOBAL, a pointer to this memory will be
404 * automatically attached to kernel globals, using the provided name
405 * matching an entry in kernel/data_arrays.h. */
406
407template<typename T> class device_vector : public device_memory {
408 public:
409 device_vector(Device *device, const char *name, MemoryType type, const uint32_t flags = 0)
410 : device_memory(device, name, type, flags)
411 {
414 modified = true;
415 need_realloc_ = true;
416
418 }
419
420 ~device_vector() override
421 {
422 free();
423 }
424
425 /* Host memory allocation. */
426 T *alloc(const size_t width, const size_t height = 0)
427 {
428 size_t new_size = size(width, height);
429
430 if (new_size != data_size) {
432 host_pointer = host_alloc(sizeof(T) * new_size);
433 modified = true;
435 }
436
437 data_size = new_size;
438 data_width = width;
439 data_height = height;
440
441 return data();
442 }
443
444 /* Host memory resize. Only use this if the original data needs to be
445 * preserved or memory needs to be initialized, it is faster to call
446 * alloc() if it can be discarded. */
447 T *resize(const size_t width, const size_t height = 0)
448 {
449 size_t new_size = size(width, height);
450
451 if (new_size != data_size) {
452 void *new_ptr = host_alloc(sizeof(T) * new_size);
453
454 if (new_ptr) {
455 size_t min_size = (new_size < data_size) ? new_size : data_size;
456 for (size_t i = 0; i < min_size; i++) {
457 ((T *)new_ptr)[i] = ((T *)host_pointer)[i];
458 }
459 for (size_t i = data_size; i < new_size; i++) {
460 ((T *)new_ptr)[i] = T();
461 }
462 }
463
465 host_pointer = new_ptr;
466 modified = true;
468 }
469
470 data_size = new_size;
471 data_width = width;
472 data_height = height;
473
474 return data();
475 }
476
477 /* Host-only resize: grows the host buffer while leaving any existing device allocation
478 * untouched. Use this when a kernel may be reading from device_pointer and freeing it
479 * would be unsafe. The device buffer will be reallocated on the next copy_to_device()
480 * call once the device is idle. Only valid when not shrinking. */
481 T *host_only_resize(const size_t new_count)
482 {
483 assert(new_count >= data_size);
484
485 if (new_count != data_size) {
486 void *new_ptr = host_alloc(sizeof(T) * new_count);
487
488 if (new_ptr) {
489 for (size_t i = 0; i < data_size; i++) {
490 ((T *)new_ptr)[i] = ((T *)host_pointer)[i];
491 }
492 for (size_t i = data_size; i < new_count; i++) {
493 ((T *)new_ptr)[i] = T();
494 }
495 }
496
498 host_pointer = new_ptr;
499 modified = true;
500 }
501
502 data_size = new_count;
503 data_width = new_count;
504
505 return data();
506 }
507
508 /* Take over data from an existing array. */
510 {
512
513 data_size = from.size();
514 data_width = 0;
515 data_height = 0;
517 modified = true;
519 }
520
521 /* Free device and host memory. */
522 void free()
523 {
525
526 data_size = 0;
527 data_width = 0;
528 data_height = 0;
529 host_pointer = 0;
530 modified = true;
531 need_realloc_ = true;
533 }
534
535 void free_if_need_realloc(bool force_free)
536 {
537 if (need_realloc_ || force_free) {
538 free();
539 }
540 }
541
542 bool is_modified() const
543 {
544 return modified;
545 }
546
548 {
549 return need_realloc_;
550 }
551
553 {
554 modified = true;
555 }
556
558 {
559 need_realloc_ = true;
560 tag_modified();
561 }
562
563 size_t size() const
564 {
565 return data_size;
566 }
567
569 {
570 return (T *)host_pointer;
571 }
572
573 const T *data() const
574 {
575 return (T *)host_pointer;
576 }
577
578 T &operator[](size_t i)
579 {
580 assert(i < data_size);
581 return data()[i];
582 }
583
585 {
586 if (data_size != 0) {
588 }
589 }
590
592 {
593 if (!modified) {
594 return;
595 }
596
598 }
599
601 {
602 modified = false;
603 need_realloc_ = false;
604 }
605
607 {
608 device_copy_from(0, data_width, (data_height == 0) ? 1 : data_height, sizeof(T));
609 }
610
611 void copy_from_device(const size_t y, const size_t w, size_t h)
612 {
613 device_copy_from(y, w, h, sizeof(T));
614 }
615
616 /* Copy from all devices and OR into host memory. */
621
623 {
624 device_zero();
625 }
626
627 protected:
628 size_t size(const size_t width, const size_t height)
629 {
630 return width * ((height == 0) ? 1 : height);
631 }
632};
633
634/* Device Sub Memory
635 *
636 * Pointer into existing memory. It is not allocated separately, but created
637 * from an already allocated base memory. It is freed automatically when it
638 * goes out of scope, which should happen before base memory is freed.
639 *
640 * NOTE: some devices require offset and size of the sub_ptr to be properly
641 * aligned to device->mem_address_alignment(). */
642
644 public:
645 device_sub_ptr(device_memory &mem, const size_t offset, const size_t size);
647
649 {
650 return ptr;
651 }
652
653 protected:
654 /* No copying. */
656
659};
660
661/* Device Image
662 *
663 * 2D or 3D image texture memory. */
664
666 public:
668 const char *name,
669 const uint image_info_id,
670 ImageDataType image_data_type,
671 InterpolationType interpolation,
672 ExtensionType extension);
673 ~device_image() override;
674
675 string log_name() const override;
676
677 void *alloc(const size_t width, const size_t height);
678
679 template<typename T = void> T *data()
680 {
681 return reinterpret_cast<T *>(host_pointer);
682 }
683
684 void copy_to_device();
685
688
689 protected:
690 size_t size(const size_t width, const size_t height)
691 {
692 return width * ((height == 0) ? 1 : height);
693 }
694};
695
unsigned long long int uint64_t
btScalar max
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
Definition btQuadWord.h:119
T * steal_pointer()
size_t size() const
void * alloc(const size_t width, const size_t height)
Definition memory.cpp:291
device_image(Device *device, const char *name, const uint image_info_id, ImageDataType image_data_type, InterpolationType interpolation, ExtensionType extension)
Definition memory.cpp:218
size_t size(const size_t width, const size_t height)
void copy_to_device()
Definition memory.cpp:313
KernelImageInfo info
string log_name() const override
Definition memory.cpp:280
friend class CUDADevice
void host_only_free()
Definition memory.cpp:126
bool is_resident(Device *sub_device) const
Definition memory.cpp:193
void * host_alloc(const size_t size)
Definition memory.cpp:93
device_memory(const device_memory &)=delete
void device_zero()
Definition memory.cpp:161
void device_alloc()
Definition memory.cpp:135
void device_copy_to()
Definition memory.cpp:141
void swap_device(Device *new_device, const size_t new_device_size, device_ptr new_device_ptr)
Definition memory.cpp:173
friend class HIPRTDevice
size_t memory_elements_size(const int elements)
const char * global_name() const
Definition memory.cpp:83
device_ptr original_device_ptr
void host_and_device_free()
Definition memory.cpp:108
bool is_shared(Device *sub_device) const
Definition memory.cpp:198
device_memory(device_memory &&other) noexcept=delete
bool device_is_cpu()
Definition memory.cpp:168
void device_move_to_host()
Definition memory.cpp:148
void restore_device()
Definition memory.cpp:186
device_memory & operator=(const device_memory &)=delete
device_memory & operator=(device_memory &&)=delete
virtual string log_name() const
Definition memory.cpp:88
void device_copy_merged_bitmap_from(const size_t y, const size_t w, size_t h)
friend class OneapiDevice
void device_copy_from(const size_t y, const size_t w, size_t h, const size_t elem)
Definition memory.cpp:155
friend class MetalDevice
friend class OptiXDevice
void alloc_to_device(const size_t num, bool shrink_to_fit=true)
device_only_memory(device_only_memory &&other) noexcept
device_only_memory(Device *device, const char *name, bool allow_host_memory_fallback=false)
device_sub_ptr(device_memory &mem, const size_t offset, const size_t size)
Definition memory.cpp:205
device_sub_ptr & operator=(const device_sub_ptr &)
device_ptr operator*() const
size_t size() const
T * alloc(const size_t width, const size_t height=0)
T * host_only_resize(const size_t new_count)
size_t size(const size_t width, const size_t height)
T * resize(const size_t width, const size_t height=0)
void copy_merged_bitmap_from_device()
device_vector(Device *device, const char *name, MemoryType type, const uint32_t flags=0)
bool is_modified() const
void steal_data(array< T > &from)
void free_if_need_realloc(bool force_free)
const T * data() const
T & operator[](size_t i)
void copy_from_device(const size_t y, const size_t w, size_t h)
Definition half.h:29
@ MEM_FLAG_PER_DEVICE
@ MEM_FLAG_NO_HOST_FALLBACK
static constexpr size_t datatype_size(DataType datatype)
@ MEM_IMAGE_TEXTURE
@ MEM_READ_WRITE
@ MEM_DEVICE_ONLY
@ MEM_READ_ONLY
@ TYPE_UNKNOWN
@ TYPE_UINT16
@ TYPE_UINT64
#define CCL_NAMESPACE_END
#define assert(assertion)
unsigned int uchar
#define T
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
static const DataType data_type
static const DataType data_type
static const DataType data_type
static const DataType data_type
static const DataType data_type
static const DataType data_type
static const size_t num_elements
static const DataType data_type
Definition half.h:53
uint64_t device_ptr
Definition types_base.h:44
ImageDataType
Definition types_image.h:35
InterpolationType
Definition types_image.h:24
ExtensionType
Definition types_image.h:80