Blender V5.3
device/device.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#include <cstdlib>
6#include <cstring>
7
8#include "bvh/bvh2.h"
9
10#include "device/device.h"
11#include "device/queue.h"
12
13#include "device/cpu/device.h"
14#include "device/cpu/kernel.h"
15#include "device/cuda/device.h"
16#include "device/dummy/device.h"
17#include "device/hip/device.h"
18#include "device/metal/device.h"
19#include "device/multi/device.h"
21#include "device/optix/device.h"
22
23#include "util/log.h"
24#include "util/math.h"
25#include "util/string.h"
26#include "util/system.h"
27#include "util/task.h"
28#include "util/types.h"
29#include "util/vector.h"
30
32
33bool Device::need_types_update = true;
34bool Device::need_devices_update = true;
35thread_mutex Device::device_mutex;
36uint Device::devices_initialized_mask = 0;
37
38/* Lazily init inside function so they get destructed before guardedalloc leak check. */
39vector<DeviceInfo> &Device::cuda_devices()
40{
41 static vector<DeviceInfo> devices_;
42 return devices_;
43}
44
45vector<DeviceInfo> &Device::optix_devices()
46{
47 static vector<DeviceInfo> devices_;
48 return devices_;
49}
50
51vector<DeviceInfo> &Device::cpu_devices()
52{
53 static vector<DeviceInfo> devices_;
54 return devices_;
55}
56
57vector<DeviceInfo> &Device::hip_devices()
58{
59 static vector<DeviceInfo> devices_;
60 return devices_;
61}
62
63vector<DeviceInfo> &Device::metal_devices()
64{
65 static vector<DeviceInfo> devices_;
66 return devices_;
67}
68
69vector<DeviceInfo> &Device::oneapi_devices()
70{
71 static vector<DeviceInfo> devices_;
72 return devices_;
73}
74
75/* Device */
76
77Device::~Device() noexcept(false) = default;
78
79void Device::set_error(const string &error)
80{
81 if (!have_error()) {
82 error_msg = error;
83 }
84 LOG_ERROR << error;
85 fflush(stderr);
86}
87
88void Device::build_bvh(BVH *bvh, Progress &progress, bool refit)
89{
91
92 BVH2 *const bvh2 = static_cast<BVH2 *>(bvh);
93 if (refit) {
94 bvh2->refit(progress);
95 }
96 else {
97 bvh2->build(progress, &stats);
98 }
99}
100
102 Stats &stats,
104 bool headless)
105{
106 if (!info.multi_devices.empty()) {
107 /* Always create a multi device when info contains multiple devices.
108 * This is done so that the type can still be e.g. DEVICE_CPU to indicate
109 * that it is a homogeneous collection of devices, which simplifies checks. */
111 }
112
113 unique_ptr<Device> device;
114
115 switch (info.type) {
116 case DEVICE_CPU:
118 break;
119#ifdef WITH_CUDA
120 case DEVICE_CUDA:
121 if (device_cuda_init()) {
123 }
124 break;
125#endif
126#ifdef WITH_OPTIX
127 case DEVICE_OPTIX:
128 if (device_optix_init()) {
130 }
131 break;
132#endif
133
134#ifdef WITH_HIP
135 case DEVICE_HIP:
136 if (device_hip_init()) {
138 }
139 break;
140#endif
141
142#ifdef WITH_METAL
143 case DEVICE_METAL:
144 if (device_metal_init()) {
146 }
147 break;
148#endif
149
150#ifdef WITH_ONEAPI
151 case DEVICE_ONEAPI:
153 break;
154#endif
155
156 default:
157 break;
158 }
159
160 if (device == nullptr) {
162 }
163
164 return device;
165}
166
168{
169 if (strcmp(name, "CPU") == 0) {
170 return DEVICE_CPU;
171 }
172 if (strcmp(name, "CUDA") == 0) {
173 return DEVICE_CUDA;
174 }
175 if (strcmp(name, "OPTIX") == 0) {
176 return DEVICE_OPTIX;
177 }
178 if (strcmp(name, "MULTI") == 0) {
179 return DEVICE_MULTI;
180 }
181 if (strcmp(name, "HIP") == 0) {
182 return DEVICE_HIP;
183 }
184 if (strcmp(name, "METAL") == 0) {
185 return DEVICE_METAL;
186 }
187 if (strcmp(name, "ONEAPI") == 0) {
188 return DEVICE_ONEAPI;
189 }
190 if (strcmp(name, "HIPRT") == 0) {
191 return DEVICE_HIPRT;
192 }
193
194 return DEVICE_NONE;
195}
196
198{
199 if (type == DEVICE_CPU) {
200 return "CPU";
201 }
202 if (type == DEVICE_CUDA) {
203 return "CUDA";
204 }
205 if (type == DEVICE_OPTIX) {
206 return "OPTIX";
207 }
208 if (type == DEVICE_MULTI) {
209 return "MULTI";
210 }
211 if (type == DEVICE_HIP) {
212 return "HIP";
213 }
214 if (type == DEVICE_METAL) {
215 return "METAL";
216 }
217 if (type == DEVICE_ONEAPI) {
218 return "ONEAPI";
219 }
220 if (type == DEVICE_HIPRT) {
221 return "HIPRT";
222 }
223
224 return "";
225}
226
228{
229 vector<DeviceType> types;
230 types.push_back(DEVICE_CPU);
231#ifdef WITH_CUDA
232 types.push_back(DEVICE_CUDA);
233#endif
234#ifdef WITH_OPTIX
235 types.push_back(DEVICE_OPTIX);
236#endif
237#ifdef WITH_HIP
238 types.push_back(DEVICE_HIP);
239#endif
240#ifdef WITH_METAL
241 types.push_back(DEVICE_METAL);
242#endif
243#ifdef WITH_ONEAPI
244 types.push_back(DEVICE_ONEAPI);
245#endif
246#ifdef WITH_HIPRT
247 types.push_back(DEVICE_HIPRT);
248#endif
249 return types;
250}
251
253{
254 /* Lazy initialize devices. On some platforms OpenCL or CUDA drivers can
255 * be broken and cause crashes when only trying to get device info, so
256 * we don't want to do any initialization until the user chooses to. */
257 const thread_scoped_lock lock(device_mutex);
258 vector<DeviceInfo> devices;
259
260#if defined(WITH_CUDA) || defined(WITH_OPTIX)
262 if (!(devices_initialized_mask & DEVICE_MASK_CUDA)) {
263 if (device_cuda_init()) {
264 device_cuda_info(cuda_devices());
265 }
266 devices_initialized_mask |= DEVICE_MASK_CUDA;
267 }
268 if (mask & DEVICE_MASK_CUDA) {
269 for (DeviceInfo &info : cuda_devices()) {
270 devices.push_back(info);
271 }
272 }
273 }
274#endif
275
276#ifdef WITH_OPTIX
277 if (mask & DEVICE_MASK_OPTIX) {
278 if (!(devices_initialized_mask & DEVICE_MASK_OPTIX)) {
279 bool meets_nvidia_driver_requirement = true;
280 if (device_optix_init(&meets_nvidia_driver_requirement) || !meets_nvidia_driver_requirement)
281 {
282 device_optix_info(cuda_devices(), optix_devices());
283 for (DeviceInfo &info : optix_devices()) {
284 info.meets_driver_requirement &= meets_nvidia_driver_requirement;
285 }
286 }
287 else {
288 /* `device_optix_init` has failed but not because of the driver being too old.
289 * Nothing to do in this case. */
290 }
291 devices_initialized_mask |= DEVICE_MASK_OPTIX;
292 }
293 for (DeviceInfo &info : optix_devices()) {
294 devices.push_back(info);
295 }
296 }
297#endif
298
299#ifdef WITH_HIP
300 if (mask & DEVICE_MASK_HIP) {
301 if (!(devices_initialized_mask & DEVICE_MASK_HIP)) {
302 bool meets_amd_driver_requirement = true;
303 if (device_hip_init(&meets_amd_driver_requirement)) {
304 device_hip_info(hip_devices());
305 for (DeviceInfo &info : hip_devices()) {
306 info.meets_driver_requirement = meets_amd_driver_requirement;
307 }
308 }
309 else if (meets_amd_driver_requirement == false) {
310 /* If we are here, then hipewInit has failed with HIPEW_ERROR_OLD_DRIVER. */
311 /* It is unclear if proper device info can be collected at this point, so we create
312 * a placeholder device to communicate the need to upgrade the driver, as presumably
313 * the hardware is available. */
316 info.description = "Unknown AMD device";
317 info.id = "unknown_amd_device_with_outdated_driver";
318 info.num = 0;
320 hip_devices().push_back(info);
321 }
322 else {
323 /* `device_hip_init` has failed but not because of the driver being too old.
324 * Nothing to do in this case. */
325 }
326 devices_initialized_mask |= DEVICE_MASK_HIP;
327 }
328 for (DeviceInfo &info : hip_devices()) {
329 devices.push_back(info);
330 }
331 }
332#endif
333
334#ifdef WITH_ONEAPI
335 if (mask & DEVICE_MASK_ONEAPI) {
336 if (!(devices_initialized_mask & DEVICE_MASK_ONEAPI)) {
337 if (device_oneapi_init()) {
338 device_oneapi_info(oneapi_devices());
339 }
340 devices_initialized_mask |= DEVICE_MASK_ONEAPI;
341 }
342 for (DeviceInfo &info : oneapi_devices()) {
343 devices.push_back(info);
344 }
345 }
346#endif
347
348 if (mask & DEVICE_MASK_CPU) {
349 if (!(devices_initialized_mask & DEVICE_MASK_CPU)) {
350 device_cpu_info(cpu_devices());
351 devices_initialized_mask |= DEVICE_MASK_CPU;
352 }
353 for (const DeviceInfo &info : cpu_devices()) {
354 devices.push_back(info);
355 }
356 }
357
358#ifdef WITH_METAL
359 if (mask & DEVICE_MASK_METAL) {
360 if (!(devices_initialized_mask & DEVICE_MASK_METAL)) {
361 if (device_metal_init()) {
362 device_metal_info(metal_devices());
363 }
364 devices_initialized_mask |= DEVICE_MASK_METAL;
365 }
366 for (const DeviceInfo &info : metal_devices()) {
367 devices.push_back(info);
368 }
369 }
370#endif
371
372 return devices;
373}
374
382
384{
385 const thread_scoped_lock lock(device_mutex);
386 string capabilities;
387
388 if (mask & DEVICE_MASK_CPU) {
389 capabilities += "\nCPU device capabilities: ";
390 capabilities += device_cpu_capabilities() + "\n";
391 }
392
393#ifdef WITH_CUDA
394 if (mask & DEVICE_MASK_CUDA) {
395 if (device_cuda_init()) {
397 if (!device_capabilities.empty()) {
398 capabilities += "\nCUDA device capabilities:\n";
399 capabilities += device_capabilities;
400 }
401 }
402 }
403#endif
404
405#ifdef WITH_HIP
406 if (mask & DEVICE_MASK_HIP) {
407 if (device_hip_init()) {
409 if (!device_capabilities.empty()) {
410 capabilities += "\nHIP device capabilities:\n";
411 capabilities += device_capabilities;
412 }
413 }
414 }
415#endif
416
417#ifdef WITH_ONEAPI
418 if (mask & DEVICE_MASK_ONEAPI) {
419 if (device_oneapi_init()) {
421 if (!device_capabilities.empty()) {
422 capabilities += "\noneAPI device capabilities:\n";
423 capabilities += device_capabilities;
424 }
425 }
426 }
427#endif
428
429#ifdef WITH_METAL
430 if (mask & DEVICE_MASK_METAL) {
431 if (device_metal_init()) {
433 if (!device_capabilities.empty()) {
434 capabilities += "\nMetal device capabilities:\n";
435 capabilities += device_capabilities;
436 }
437 }
438 }
439#endif
440
441 return capabilities;
442}
443
445 const int threads,
446 bool background)
447{
448 assert(!subdevices.empty());
449
450 if (subdevices.size() == 1) {
451 /* No multi device needed. */
452 return subdevices.front();
453 }
454
457 info.id = "MULTI";
458 info.description = "Multi Device";
459 info.num = 0;
460
461 info.has_nanovdb = true;
462 info.has_mnee_ = true;
463 info.has_osl = true;
464 info.has_guiding = true;
465 info.has_profiling = true;
466 info.has_peer_memory = false;
469
470 for (const DeviceInfo &device : subdevices) {
471 /* Ensure CPU device does not slow down GPU. */
472 if (device.type == DEVICE_CPU && subdevices.size() > 1) {
473 if (background) {
474 const int orig_cpu_threads = (threads) ? threads : TaskScheduler::max_concurrency();
475 const int cpu_threads = max(orig_cpu_threads - (subdevices.size() - 1), size_t(0));
476
477 LOG_INFO << "CPU render threads reduced from " << orig_cpu_threads << " to " << cpu_threads
478 << ", to dedicate to GPU.";
479
480 if (cpu_threads >= 1) {
481 DeviceInfo cpu_device = device;
482 cpu_device.cpu_threads = cpu_threads;
483 info.multi_devices.push_back(cpu_device);
484 }
485 else {
486 continue;
487 }
488 }
489 else {
490 LOG_INFO << "CPU render threads disabled for interactive render.";
491 continue;
492 }
493 }
494 else {
495 info.multi_devices.push_back(device);
496 }
497
498 /* Create unique ID for this combination of devices. */
499 info.id += device.id;
500
501 /* Set device type to MULTI if subdevices are not of a common type. */
502 if (info.type == DEVICE_NONE) {
503 info.type = device.type;
504 }
505 else if (device.type != info.type) {
507 }
508
509 /* Accumulate device info. */
510 info.has_nanovdb &= device.has_nanovdb;
511 info.has_mnee_ &= device.has_mnee_;
512 info.has_osl &= device.has_osl;
513 info.has_guiding &= device.has_guiding;
514 info.has_profiling &= device.has_profiling;
515 info.has_peer_memory |= device.has_peer_memory;
516 info.use_hardware_raytracing |= device.use_hardware_raytracing;
517 info.denoisers &= device.denoisers;
518 }
519
520 return info;
521}
522
524{
525 free_memory();
526}
527
529{
530 devices_initialized_mask = 0;
531 cuda_devices().free_memory();
532 optix_devices().free_memory();
533 hip_devices().free_memory();
534 oneapi_devices().free_memory();
535 cpu_devices().free_memory();
536 metal_devices().free_memory();
537}
538
540{
541 LOG_FATAL << "Device does not support queues.";
542 return nullptr;
543}
544
546{
547 /* Initialize CPU kernels once and reuse. */
548 static const CPUKernels kernels;
549 return kernels;
550}
551
553{
554 LOG_FATAL << "Device does not support CPU kernels.";
555 return nullptr;
556}
557
559{
560 /* No-op for non-CPU devices. */
561}
562
564{
565 return nullptr;
566}
567
569{
570 LOG_ERROR << "Request guiding field from a device which does not support it.";
571 return nullptr;
572}
573
574void *Device::host_alloc(const MemoryType /*type*/, const size_t size)
575{
577}
578
579void Device::host_free(const MemoryType /*type*/, void *host_pointer, const size_t size)
580{
581 util_aligned_free(host_pointer, size);
582}
583
585{
586 /* Note that we always accumulate into the host buffer without zeroing, as CPU and unified
587 * memory write into the host buffer and we need to combine with those flags. */
588 const size_t size = mem.memory_size();
589 vector<uint8_t> tmp(size);
590 uint8_t *combined = static_cast<uint8_t *>(mem.host_pointer);
591 mem.host_pointer = tmp.data();
593 mem, 0, mem.data_width, (mem.data_height == 0) ? 1 : mem.data_height, sizeof(uint8_t));
594 const uint8_t *src = (const uint8_t *)mem.host_pointer;
595 for (size_t i = 0; i < size; i++) {
596 combined[i] |= src[i];
597 }
598 mem.host_pointer = combined;
599}
600
602{
603 assert(sub_device == this);
604 (void)sub_device;
605 return mem.device_pointer;
606}
607
608GPUDevice::~GPUDevice() noexcept(false) = default;
609
611{
612 /* Note image_info is never host mapped, and load_image_info() should only
613 * be called right before kernel enqueue when all memory operations have completed. */
614
615 /* Guard against concurrent #GPUDevice::image_info_update() resizes. */
617
618 {
619 /* Clear the flag before copying and under a lock, multiple GPU host threads
620 * may call this function in parallel. */
622 if (!need_image_info) {
623 return false;
624 }
625 need_image_info = false;
626 }
627
628 /* If the host buffer was grown with host_only_resize() while a kernel was reading the old
629 * device buffer, we now free and reallocate it. */
630 if (image_info.device_size < image_info.memory_size()) {
632 }
633 if (queue) {
634 queue->copy_to_device(image_info);
635 }
636 else {
637 image_info.copy_to_device();
638 }
639 return true;
640}
641
642void GPUDevice::image_info_update(const uint image_info_id, const KernelImageInfo &info)
643{
645
646 if (image_info_id >= image_info.size()) {
647 /* Wait for GPUDevice::load_image_info() to finish copying. */
648 lock.unlock();
650 lock.lock();
651
652 if (image_info_id >= image_info.size()) {
653 /* Geometric growth to amortize reallocation cost. */
654 const size_t new_size = max(size_t(image_info_id) + 128, image_info.size() * 2);
656 }
657 }
658
659 image_info[image_info_id] = info;
660 need_image_info = true;
661}
662
663void GPUDevice::init_host_memory(const size_t preferred_texture_headroom,
664 const size_t preferred_working_headroom)
665{
666 /* Limit amount of host mapped memory, because allocating too much can
667 * cause system instability. Leave at least half or 4 GB of system
668 * memory free, whichever is smaller. */
669 const size_t default_limit = 4 * 1024 * 1024 * 1024LL;
670 const size_t system_ram = system_physical_ram();
671
672 if (system_ram > 0) {
673 if (system_ram / 2 > default_limit) {
674 map_host_limit = system_ram - default_limit;
675 }
676 else {
677 map_host_limit = system_ram / 2;
678 }
679 }
680 else {
681 LOG_WARNING << "Mapped host memory disabled, failed to get system RAM";
682 map_host_limit = 0;
683 }
684
685 /* Amount of device memory to keep free after texture memory
686 * and working memory allocations respectively. We set the working
687 * memory limit headroom lower than the working one so there
688 * is space left for it. */
689 device_working_headroom = preferred_working_headroom > 0 ? preferred_working_headroom :
690 32 * 1024 * 1024LL; // 32MB
691 device_image_headroom = preferred_texture_headroom > 0 ? preferred_texture_headroom :
692 128 * 1024 * 1024LL; // 128MB
693
694 LOG_INFO << "Mapped host memory limit set to " << string_human_readable_number(map_host_limit)
695 << " bytes. (" << string_human_readable_size(map_host_limit) << ")";
696}
697
698void GPUDevice::move_textures_to_host(size_t size, const size_t headroom, const bool for_texture)
699{
700 static thread_mutex move_mutex;
701 const thread_scoped_lock lock(move_mutex);
702
703 /* Check if there is enough space. Within mutex locks so that multiple threads
704 * calling take into account memory freed by another thread. */
705 size_t total = 0;
706 size_t free = 0;
707 get_device_memory_info(total, free);
708 if (size + headroom < free) {
709 return;
710 }
711
712 while (size > 0) {
713 /* Find suitable memory allocation to move. */
714 device_memory *max_mem = nullptr;
715 size_t max_size = 0;
716 bool max_is_image = false;
717
719 for (MemMap::value_type &pair : device_mem_map) {
720 device_memory &mem = *pair.first;
721 Mem *cmem = &pair.second;
722
723 /* Can only move textures allocated on this device (and not those from peer devices).
724 * And need to ignore memory that is already on the host. */
725 if (!mem.is_resident(this) || mem.is_shared(this)) {
726 continue;
727 }
728
729 const bool is_texture = (mem.type == MEM_IMAGE_TEXTURE || mem.type == MEM_GLOBAL) &&
730 (&mem != &image_info);
731 const bool is_image = is_texture && (mem.data_height > 1);
732
733 /* Can't move this type of memory. */
734 if (!is_texture || cmem->array || (mem.flags & MEM_FLAG_NO_HOST_FALLBACK)) {
735 continue;
736 }
737
738 /* For other textures, only move image textures. */
739 if (for_texture && !is_image) {
740 continue;
741 }
742
743 /* Try to move largest allocation, prefer moving images. */
744 if (is_image > max_is_image || (is_image == max_is_image && mem.device_size > max_size)) {
745 max_is_image = is_image;
746 max_size = mem.device_size;
747 max_mem = &mem;
748 }
749 }
750 lock.unlock();
751
752 /* Move to host memory. This part is mutex protected since
753 * multiple backend devices could be moving the memory. The
754 * first one will do it, and the rest will adopt the pointer. */
755 if (max_mem) {
756 LOG_DEBUG << "Move memory from device to host: " << max_mem->log_name();
757
758 /* Potentially need to call back into multi device, so pointer mapping
759 * and peer devices are updated. This is also necessary since the device
760 * pointer may just be a key here, so cannot be accessed and freed directly.
761 * Unfortunately it does mean that memory is reallocated on all other
762 * devices as well, which is potentially dangerous when still in use (since
763 * a thread rendering on another devices would only be caught in this mutex
764 * if it so happens to do an allocation at the same time as well. */
765 max_mem->move_to_host = true;
766 max_mem->device_move_to_host();
767 max_mem->move_to_host = false;
768 size = (max_size >= size) ? 0 : size - max_size;
769
770 /* Tag image info update for new pointers. */
771 need_image_info = true;
772 }
773 else {
774 break;
775 }
776 }
777}
778
779GPUDevice::Mem *GPUDevice::generic_alloc(device_memory &mem, const size_t pitch_padding)
780{
781 void *device_pointer = nullptr;
782 const size_t size = mem.memory_size() + pitch_padding;
783
784 bool mem_alloc_result = false;
785 const char *status = "";
786
787 /* First try allocating in device memory, respecting headroom. We make
788 * an exception for image info. It is small and frequently accessed,
789 * so treat it as working memory.
790 *
791 * If there is not enough room for other working memory, we will try to move
792 * textures to host memory, assuming the performance impact would have
793 * been worse for working memory. */
794 const bool is_texture = (mem.type == MEM_IMAGE_TEXTURE || mem.type == MEM_GLOBAL) &&
795 (&mem != &image_info);
796 const bool is_image = is_texture && (mem.data_height > 1);
797
798 const size_t headroom = (is_texture) ? device_image_headroom : device_working_headroom;
799
800 const bool no_host_fallback = (mem.type == MEM_DEVICE_ONLY) ||
802
803 if (!mem.move_to_host) {
804 /* Move textures to host memory if needed.
805 *
806 * Not for image_info, it's reallocated right before kernel enqueue while
807 * other devices may be using the textures. Without host fallback it may use
808 * the working memory headroom, which is much larger than image_info. */
809 if (!is_image && can_map_host && &mem != &image_info) {
810 move_textures_to_host(size, headroom, is_texture);
811 }
812
813 size_t total = 0;
814 size_t free = 0;
815 get_device_memory_info(total, free);
816
817 /* Allocate in device memory. */
818 if ((size + headroom) < free || no_host_fallback) {
819 mem_alloc_result = alloc_device(device_pointer, size);
820 if (mem_alloc_result) {
821 status = " in device memory";
822 }
823 }
824 }
825
826 /* Fall back to mapped host memory if needed and possible. */
827
828 void *shared_pointer = nullptr;
829
830 if (!mem_alloc_result && can_map_host && !no_host_fallback) {
831 if (mem.shared_pointer) {
832 /* Another device already allocated host memory. */
833 mem_alloc_result = true;
834 shared_pointer = mem.shared_pointer;
835 }
836 else if (map_host_used + size < map_host_limit) {
837 /* Allocate host memory ourselves. */
838 mem_alloc_result = shared_alloc(shared_pointer, size);
839
840 assert((mem_alloc_result && shared_pointer != nullptr) ||
841 (!mem_alloc_result && shared_pointer == nullptr));
842 }
843
844 if (mem_alloc_result) {
845 device_pointer = shared_to_device_pointer(shared_pointer);
846 map_host_used += size;
847 status = " in host memory";
848 }
849 }
850
851 if (!mem_alloc_result) {
852 if (no_host_fallback) {
853 status = " failed, out of device memory";
854 set_error("System is out of GPU memory");
855 }
856 else {
857 status = " failed, out of device and host memory";
858 set_error("System is out of GPU and shared host memory");
859 }
860 }
861
862 LOG_DEBUG << "Buffer allocate: " << mem.log_name() << ", "
863 << string_human_readable_number(mem.memory_size()) << " bytes. ("
864 << string_human_readable_size(mem.memory_size()) << ")" << status;
865
866 mem.device_pointer = (device_ptr)device_pointer;
867 mem.device_size = size;
868 stats.mem_alloc(size);
869
870 if (!mem.device_pointer) {
871 return nullptr;
872 }
873
874 /* Insert into map of allocations. */
876 Mem *cmem = &device_mem_map[&mem];
877 if (shared_pointer != nullptr) {
878 /* Replace host pointer with our host allocation. Only works if
879 * memory layout is the same and has no pitch padding. Also
880 * does not work if we move textures to host during a render,
881 * since other devices might be using the memory. */
882
883 if (!mem.move_to_host && pitch_padding == 0 && mem.host_pointer &&
884 mem.host_pointer != shared_pointer)
885 {
886 memcpy(shared_pointer, mem.host_pointer, size);
887 host_free(mem.type, mem.host_pointer, mem.memory_size());
888 mem.host_pointer = shared_pointer;
889 }
890 mem.shared_pointer = shared_pointer;
891 mem.shared_counter++;
892 }
893
894 return cmem;
895}
896
898{
899 if (!(mem.device_pointer && mem.is_resident(this))) {
900 return;
901 }
902
903 /* Host pointer should already have been freed at this point. If not we might
904 * end up freeing shared memory and can't recover original host memory. */
905 assert(mem.host_pointer == nullptr || mem.move_to_host || !mem.is_shared(this));
906
908 DCHECK(device_mem_map.find(&mem) != device_mem_map.end());
909
910 /* For host mapped memory, reference counting is used to safely free it. */
911 if (mem.is_shared(this)) {
912 assert(mem.shared_counter > 0);
913 if (--mem.shared_counter == 0) {
914 if (mem.host_pointer == mem.shared_pointer) {
915 /* Safely move the device-side data back to the host before it is freed.
916 * We should actually never reach this code as it is inefficient, but
917 * better than to crash if there is a bug. */
918 assert(!"GPU device should not copy memory back to host");
919 const size_t size = mem.memory_size();
920 mem.host_pointer = mem.host_alloc(size);
921 memcpy(mem.host_pointer, mem.shared_pointer, size);
922 }
924 mem.shared_pointer = nullptr;
925 }
927 }
928 else {
929 /* Free device memory. */
930 free_device((void *)mem.device_pointer);
931 }
932
934 mem.device_pointer = 0;
935 mem.device_size = 0;
936
937 device_mem_map.erase(device_mem_map.find(&mem));
938}
939
941{
942 if (!mem.host_pointer || !mem.device_pointer) {
943 return;
944 }
945
946 /* If not host mapped, the current device only uses device memory allocated by backend
947 * device allocation regardless of mem.host_pointer and mem.shared_pointer, and should
948 * copy data from mem.host_pointer. */
949 if (!(mem.is_shared(this) && mem.host_pointer == mem.shared_pointer)) {
951 }
952}
953
954bool GPUDevice::is_shared(const void *shared_pointer,
955 const device_ptr device_pointer,
956 Device * /*sub_device*/)
957{
958 return (shared_pointer && device_pointer &&
959 (device_ptr)shared_to_device_pointer(shared_pointer) == device_pointer);
960}
961
962/* DeviceInfo */
963
CCL_NAMESPACE_BEGIN void * util_aligned_malloc(const size_t size, const int alignment)
void util_aligned_free(void *ptr, const size_t size)
volatile int lock
btScalar max
void refit(btStridingMeshInterface *triangles, const btVector3 &aabbMin, const btVector3 &aabbMax)
Definition bvh2.h:38
void refit(Progress &progress)
Definition bvh2.cpp:78
void build(Progress &progress, Stats *stats)
Definition bvh2.cpp:39
BVHLayout bvh_layout
Definition params.h:83
Definition bvh/bvh.h:67
BVHParams params
Definition bvh/bvh.h:69
bool meets_driver_requirement
vector< DeviceInfo > multi_devices
DenoiserTypeMask denoisers
string error_msg
bool has_peer_memory
DeviceInfo()=default
bool has_profiling
DeviceType type
string description
bool use_hardware_raytracing
virtual void host_free(const MemoryType type, void *host_pointer, const size_t size)
virtual void * get_guiding_device() const
static void free_memory()
static DeviceInfo dummy_device(const string &error_msg="")
Device(const DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool headless_)
static DeviceInfo get_multi_device(const vector< DeviceInfo > &subdevices, const int threads, bool background)
static void tag_update()
static const CPUKernels & get_cpu_kernels()
string error_msg
friend class device_memory
virtual ~Device() noexcept(false)
virtual unique_ptr< DeviceQueue > gpu_queue_create()
Profiler & profiler
Stats & stats
virtual void build_bvh(BVH *bvh, Progress &progress, bool refit)
virtual device_ptr mem_device_ptr(const device_memory &mem, Device *sub_device)
static DeviceType type_from_string(const char *name)
virtual void set_error(const string &error)
bool headless
virtual void mem_or_from_device(device_memory &mem)
virtual void mem_copy_from(device_memory &mem, const size_t y, size_t w, const size_t h, size_t elem)=0
virtual vector< ThreadKernelGlobalsCPU > * acquire_cpu_kernel_thread_globals()
DeviceInfo info
static string device_capabilities(const uint device_type_mask=DEVICE_MASK_ALL)
static vector< DeviceType > available_types()
static string string_from_type(DeviceType type)
virtual OSLGlobals * get_cpu_osl_memory()
static vector< DeviceInfo > available_devices(const uint device_type_mask=DEVICE_MASK_ALL)
static unique_ptr< Device > create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
virtual void release_cpu_kernel_thread_globals()
bool have_error()
virtual void * host_alloc(const MemoryType type, const size_t size)
bool is_shared(const void *shared_pointer, const device_ptr device_pointer, Device *sub_device) override
virtual bool shared_alloc(void *&shared_pointer, const size_t size)=0
virtual bool load_image_info(DeviceQueue *queue)
size_t map_host_used
GPUDevice(const DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool headless_)
bool can_map_host
virtual void get_device_memory_info(size_t &total, size_t &free)=0
virtual bool alloc_device(void *&device_pointer, const size_t size)=0
size_t device_working_headroom
bool need_image_info
friend class device_memory
void image_info_update(uint image_info_id, const KernelImageInfo &info)
virtual void * shared_to_device_pointer(const void *shared_pointer)=0
virtual void generic_copy_to(device_memory &mem)
virtual void move_textures_to_host(const size_t size, const size_t headroom, const bool for_texture)
virtual void shared_free(void *shared_pointer, const size_t size)=0
thread_mutex image_info_mutex
virtual void copy_host_to_device(void *device_pointer, void *host_pointer, const size_t size)=0
device_vector< KernelImageInfo > image_info
size_t map_host_limit
virtual void free_device(void *device_pointer)=0
thread_mutex device_mem_map_mutex
virtual void generic_free(device_memory &mem)
virtual void init_host_memory(const size_t preferred_texture_headroom=0, const size_t preferred_working_headroom=0)
size_t device_image_headroom
thread_mutex image_info_resize_mutex
virtual GPUDevice::Mem * generic_alloc(device_memory &mem, const size_t pitch_padding=0)
MemMap device_mem_map
~GPUDevice() noexcept(false) override
void mem_alloc(const size_t size)
Definition util/stats.h:18
void mem_free(const size_t size)
Definition util/stats.h:24
static int max_concurrency()
Definition task.cpp:96
bool is_resident(Device *sub_device) const
Definition memory.cpp:193
void * host_alloc(const size_t size)
Definition memory.cpp:93
bool is_shared(Device *sub_device) const
Definition memory.cpp:198
void device_move_to_host()
Definition memory.cpp:148
virtual string log_name() const
Definition memory.cpp:88
size_t size() const
T * host_only_resize(const size_t new_count)
@ MEM_FLAG_NO_HOST_FALLBACK
@ MEM_IMAGE_TEXTURE
@ MEM_DEVICE_ONLY
#define MIN_ALIGNMENT_DEVICE_MEMORY
@ DENOISER_ALL
Definition denoise.h:18
void device_cpu_info(vector< DeviceInfo > &devices)
string device_cpu_capabilities()
CCL_NAMESPACE_BEGIN unique_ptr< Device > device_cpu_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
#define CCL_NAMESPACE_END
void device_cuda_info(vector< DeviceInfo > &devices)
string device_cuda_capabilities()
CCL_NAMESPACE_BEGIN bool device_cuda_init()
unique_ptr< Device > device_cuda_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
@ DEVICE_MASK_OPTIX
@ DEVICE_MASK_CPU
@ DEVICE_MASK_HIP
@ DEVICE_MASK_CUDA
@ DEVICE_MASK_METAL
@ DEVICE_MASK_ONEAPI
DeviceType
@ DEVICE_DUMMY
@ DEVICE_NONE
@ DEVICE_METAL
@ DEVICE_MULTI
@ DEVICE_CUDA
@ DEVICE_CPU
@ DEVICE_HIPRT
@ DEVICE_OPTIX
@ DEVICE_HIP
@ DEVICE_ONEAPI
unique_ptr< Device > device_dummy_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
CCL_NAMESPACE_BEGIN bool device_hip_init(bool *r_meets_driver_requirement)
void device_hip_info(vector< DeviceInfo > &devices)
string device_hip_capabilities()
unique_ptr< Device > device_hip_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, const bool headless)
string device_metal_capabilities()
Definition device.mm:141
unique_ptr< Device > device_metal_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
bool device_metal_init()
Definition device.mm:134
void device_metal_info(vector< DeviceInfo > &devices)
Definition device.mm:139
unique_ptr< Device > device_multi_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
unique_ptr< Device > device_oneapi_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
void device_oneapi_info(vector< DeviceInfo > &devices)
CCL_NAMESPACE_BEGIN bool device_oneapi_init()
string device_oneapi_capabilities()
CCL_NAMESPACE_BEGIN bool device_optix_init(bool *r_meets_driver_requirement)
void device_optix_info(const vector< DeviceInfo > &cuda_devices, vector< DeviceInfo > &devices)
unique_ptr< Device > device_optix_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
#define assert(assertion)
@ BVH_LAYOUT_BVH2
#define DCHECK(expression)
Definition log.h:135
#define LOG_DEBUG
Definition log.h:107
#define LOG_FATAL
Definition log.h:99
#define LOG_ERROR
Definition log.h:101
#define LOG_WARNING
Definition log.h:103
#define LOG_INFO
Definition log.h:106
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
unsigned int uint
Definition BKE_text.h:55
string string_human_readable_size(size_t size)
Definition string.cpp:333
string string_human_readable_number(size_t num)
Definition string.cpp:352
arrayMemObject array
size_t system_physical_ram()
Definition system.cpp:247
std::mutex thread_mutex
Definition thread.h:27
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
uint64_t device_ptr
Definition types_base.h:44