Blender V5.3
hip/device_impl.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#ifdef WITH_HIP
6
7# include <cstdio>
8# include <cstdlib>
9# include <cstring>
10# include <iomanip>
11
13
14# include "util/debug.h"
15# include "util/log.h"
16# include "util/md5.h"
17# include "util/path.h"
18# include "util/string.h"
19# include "util/system.h"
20# include "util/time.h"
21# include "util/types.h"
22
23# ifdef _WIN32
24# include "util/windows.h"
25# endif
26
28
30
32
33class HIPDevice;
34
35bool HIPDevice::have_precompiled_kernels()
36{
37 string fatbins_path = path_get("lib");
38 return path_exists(fatbins_path);
39}
40
41BVHLayoutMask HIPDevice::get_bvh_layout_mask(const uint64_t /*kernel_features*/) const
42{
43 return BVH_LAYOUT_BVH2;
44}
45
46void HIPDevice::set_error(const string &error)
47{
48 Device::set_error(error);
49
50 if (first_error) {
51 LOG_ERROR << "Refer to the Cycles GPU rendering documentation for possible solutions:\n"
52 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
53 first_error = false;
54 }
55}
56
57HIPDevice::HIPDevice(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
58 : GPUDevice(info, stats, profiler, headless)
59{
60 /* Verify that base class types can be used with specific backend types */
61 static_assert(sizeof(texMemObject) == sizeof(hipTextureObject_t));
62 static_assert(sizeof(arrayMemObject) == sizeof(hArray));
63
64 first_error = true;
65
66 hipDevId = info.num;
67 hipDevice = 0;
68 hipContext = nullptr;
69
70 hipModule = nullptr;
71
72 need_image_info = false;
73
74 pitch_alignment = 0;
75
76 /* Initialize HIP. */
77 hipError_t result = hipInit(0);
78 if (result != hipSuccess) {
79 set_error(string_printf("Failed to initialize HIP runtime (%s)", hipewErrorString(result)));
80 return;
81 }
82
83 /* Setup device and context. */
84 result = hipDeviceGet(&hipDevice, hipDevId);
85 if (result != hipSuccess) {
86 set_error(string_printf("Failed to get HIP device handle from ordinal (%s)",
87 hipewErrorString(result)));
88 return;
89 }
90
91 /* hipDeviceMapHost for mapping host memory when out of device memory.
92 * hipDeviceLmemResizeToMax for reserving local memory ahead of render,
93 * so we can predict which memory to map to host. */
94 int value;
95 hip_assert(hipDeviceGetAttribute(&value, hipDeviceAttributeCanMapHostMemory, hipDevice));
96 can_map_host = value != 0;
97
98 hip_assert(
99 hipDeviceGetAttribute(&pitch_alignment, hipDeviceAttributeTexturePitchAlignment, hipDevice));
100
101 unsigned int ctx_flags = hipDeviceLmemResizeToMax;
102 if (can_map_host) {
103 ctx_flags |= hipDeviceMapHost;
104 init_host_memory();
105 }
106
107 /* Create context. */
108 result = hipCtxCreate(&hipContext, ctx_flags, hipDevice);
109
110 if (result != hipSuccess) {
111 set_error(string_printf("Failed to create HIP context (%s)", hipewErrorString(result)));
112 return;
113 }
114
115 int major, minor;
116 hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, hipDevId);
117 hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, hipDevId);
118 hipDevArchitecture = major * 100 + minor * 10;
119
120 /* Get hip runtime Version needed for memory types. */
121 hip_assert(hipRuntimeGetVersion(&hipRuntimeVersion));
122
123 /* Pop context set by hipCtxCreate. */
124 hipCtxPopCurrent(nullptr);
125}
126
127HIPDevice::~HIPDevice()
128{
129 image_info.free();
130 if (hipModule) {
131 hip_assert(hipModuleUnload(hipModule));
132 }
133 hip_assert(hipCtxDestroy(hipContext));
134}
135
136bool HIPDevice::support_device(const uint64_t /*kernel_features*/)
137{
138 if (hipSupportsDevice(hipDevId)) {
139 return true;
140 }
141 /* We only support Navi and above. */
142 hipDeviceProp_t props;
143 hipGetDeviceProperties(&props, hipDevId);
144
145 set_error(string_printf("HIP backend requires AMD RDNA graphics card or up, but found %s.",
146 props.name));
147 return false;
148}
149
150bool HIPDevice::check_peer_access(Device *peer_device)
151{
152 if (peer_device == this) {
153 return false;
154 }
155 if (peer_device->info.type != DEVICE_HIP && peer_device->info.type != DEVICE_OPTIX) {
156 return false;
157 }
158
159 HIPDevice *const peer_device_hip = static_cast<HIPDevice *>(peer_device);
160
161 int can_access = 0;
162 hip_assert(hipDeviceCanAccessPeer(&can_access, hipDevice, peer_device_hip->hipDevice));
163 if (can_access == 0) {
164 return false;
165 }
166
167 // Ensure array access over the link is possible as well (for 3D images)
168 hip_assert(hipDeviceGetP2PAttribute(
169 &can_access, hipDevP2PAttrHipArrayAccessSupported, hipDevice, peer_device_hip->hipDevice));
170 if (can_access == 0) {
171 return false;
172 }
173
174 // Enable peer access in both directions
175 {
176 const HIPContextScope scope(this);
177 hipError_t result = hipCtxEnablePeerAccess(peer_device_hip->hipContext, 0);
178 if (result != hipSuccess) {
179 set_error(string_printf("Failed to enable peer access on HIP context (%s)",
180 hipewErrorString(result)));
181 return false;
182 }
183 }
184 {
185 const HIPContextScope scope(peer_device_hip);
186 hipError_t result = hipCtxEnablePeerAccess(hipContext, 0);
187 if (result != hipSuccess) {
188 set_error(string_printf("Failed to enable peer access on HIP context (%s)",
189 hipewErrorString(result)));
190 return false;
191 }
192 }
193
194 return true;
195}
196
197bool HIPDevice::use_adaptive_compilation()
198{
200}
201
202/* Common HIPCC flags which stays the same regardless of shading model,
203 * kernel sources md5 and only depends on compiler or compilation settings.
204 */
205string HIPDevice::compile_kernel_get_common_cflags(const uint64_t kernel_features)
206{
207 const int machine = system_cpu_bits();
208 const string source_path = path_get("source");
209 const string include_path = source_path;
210 string cflags = string_printf(
211 "-std=c++17 "
212 "-m%d "
213 "-DHIPCC "
214 "-I\"%s\"",
215 machine,
216 include_path.c_str());
217 if (use_adaptive_compilation()) {
218 cflags += " -D__KERNEL_FEATURES__=" + to_string(kernel_features);
219 }
220
221 const char *extra_cflags = getenv("CYCLES_HIP_EXTRA_CFLAGS");
222 if (extra_cflags) {
223 cflags += string(" ") + string(extra_cflags);
224 }
225
226# ifdef WITH_NANOVDB
227 cflags += " -DWITH_NANOVDB";
228# endif
229
230# ifdef WITH_CYCLES_DEBUG
231 cflags += " -DWITH_CYCLES_DEBUG";
232# endif
233
234 return cflags;
235}
236
237string HIPDevice::compile_kernel(const uint64_t kernel_features,
238 const char *name,
239 const char *base)
240{
241 /* Compute kernel name. */
242 int major, minor;
243 hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, hipDevId);
244 hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, hipDevId);
245 const std::string arch = hipDeviceArch(hipDevId);
246
247 /* Attempt to use kernel provided with Blender. */
248 if (!use_adaptive_compilation()) {
249 const string fatbin = path_get(string_printf("lib/%s_%s.fatbin.zst", name, arch.c_str()));
250 LOG_INFO << "Testing for pre-compiled kernel " << fatbin << ".";
251 if (path_exists(fatbin)) {
252 LOG_INFO << "Using precompiled kernel.";
253 return fatbin;
254 }
255 }
256
257 /* Try to use locally compiled kernel. */
258 string source_path = path_get("source");
259 const string source_md5 = path_files_md5_hash(source_path);
260
261 /* We include cflags into md5 so changing hip toolkit or changing other
262 * compiler command line arguments makes sure fatbin gets re-built.
263 */
264 string common_cflags = compile_kernel_get_common_cflags(kernel_features);
265 const string kernel_md5 = util_md5_string(source_md5 + common_cflags);
266
267 const char *const kernel_ext = "genco";
268 std::string options = "-Wno-parentheses-equality -Wno-unused-value -ffast-math -std=c++17";
269
270# ifndef NDEBUG
271 options.append(" -save-temps");
272# endif
273 if (major == 9 && minor == 0) {
274 /* Reduce optimization level on VEGA GPUs to avoid some rendering artifacts */
275 options.append(" -O1");
276 }
277 options.append(" --offload-arch=").append(arch);
278
279 const string include_path = source_path;
280 const string fatbin_file = string_printf(
281 "cycles_%s_%s_%s", name, arch.c_str(), kernel_md5.c_str());
282 const string fatbin = path_cache_get(path_join("kernels", fatbin_file));
283 LOG_INFO << "Testing for locally compiled kernel " << fatbin << ".";
284 if (path_exists(fatbin)) {
285 LOG_INFO << "Using locally compiled kernel.";
286 return fatbin;
287 }
288
289# ifdef _WIN32
290 if (!use_adaptive_compilation() && have_precompiled_kernels()) {
291 if (!hipSupportsDevice(hipDevId)) {
292 set_error(
293 string_printf("HIP backend requires compute capability 10.1 or up, but found %d.%d. "
294 "Your GPU is not supported.",
295 major,
296 minor));
297 }
298 else {
299 set_error(
300 string_printf("HIP binary kernel for this graphics card compute "
301 "capability (%d.%d) not found.",
302 major,
303 minor));
304 }
305 return string();
306 }
307# endif
308
309 /* Compile. */
310 const char *const hipcc = hipewCompilerPath();
311 if (hipcc == nullptr) {
312 set_error(
313 "HIP hipcc compiler not found. "
314 "Install HIP toolkit in default location.");
315 return string();
316 }
317
318 const int hipcc_hip_version = hipewCompilerVersion();
319 LOG_INFO << "Found hipcc " << hipcc << ", HIP version " << hipcc_hip_version << ".";
320
321 double starttime = time_dt();
322
324
325 source_path = path_join(path_join(source_path, "kernel"),
326 path_join("device", path_join(base, string_printf("%s.cpp", name))));
327
328 string command = string_printf("%s %s -I \"%s\" --%s \"%s\" -o \"%s\" %s",
329 hipcc,
330 options.c_str(),
331 include_path.c_str(),
332 kernel_ext,
333 source_path.c_str(),
334 fatbin.c_str(),
335 common_cflags.c_str());
336
337 LOG_INFO_IMPORTANT << "Compiling " << ((use_adaptive_compilation()) ? "adaptive " : "")
338 << "HIP kernel ... " << command;
339
340# ifdef _WIN32
341 command = "call " + command;
342# endif
343 if (system(command.c_str()) != 0) {
344 set_error(
345 "Failed to execute compilation command, "
346 "see console for details.");
347 return string();
348 }
349
350 /* Verify if compilation succeeded */
351 if (!path_exists(fatbin)) {
352 set_error(
353 "HIP kernel compilation failed, "
354 "see console for details.");
355 return string();
356 }
357
358 LOG_INFO_IMPORTANT << "Kernel compilation finished in " << std::fixed << std::setprecision(2)
359 << time_dt() - starttime << "s";
360
361 return fatbin;
362}
363
364bool HIPDevice::load_kernels(const uint64_t kernel_features)
365{
366 /* TODO(sergey): Support kernels re-load for HIP devices adaptive compile.
367 *
368 * Currently re-loading kernels will invalidate memory pointers.
369 */
370 if (hipModule) {
371 if (use_adaptive_compilation()) {
372 LOG_INFO << "Skipping HIP kernel reload for adaptive compilation, not currently supported.";
373 }
374 return true;
375 }
376
377 /* check if hip init succeeded */
378 if (hipContext == nullptr) {
379 return false;
380 }
381
382 /* check if GPU is supported */
383 if (!support_device(kernel_features)) {
384 return false;
385 }
386
387 /* get kernel */
388 const char *kernel_name = "kernel";
389 string fatbin = compile_kernel(kernel_features, kernel_name);
390 if (fatbin.empty()) {
391 return false;
392 }
393
394 /* open module */
395 HIPContextScope scope(this);
396
397 string fatbin_data;
398 hipError_t result;
399
400 if (path_read_compressed_text(fatbin, fatbin_data)) {
401 result = hipModuleLoadData(&hipModule, fatbin_data.c_str());
402 }
403 else {
404 result = hipErrorFileNotFound;
405 }
406
407 if (result != hipSuccess) {
408 set_error(string_printf(
409 "Failed to load HIP kernel from '%s' (%s)", fatbin.c_str(), hipewErrorString(result)));
410 }
411
412 if (result == hipSuccess) {
413 kernels.load_all(this, hipModule);
414 reserve_local_memory(kernel_features);
415 }
416
417 return (result == hipSuccess);
418}
419
420void HIPDevice::reserve_local_memory(const uint64_t kernel_features)
421{
422 /* Together with hipDeviceLmemResizeToMax, this reserves local memory
423 * needed for kernel launches, so that we can reliably figure out when
424 * to allocate scene data in mapped host memory. */
425 size_t total = 0, free_before = 0, free_after = 0;
426
427 {
428 HIPContextScope scope(this);
429 hipMemGetInfo(&free_before, &total);
430 }
431
432 {
433 /* Use the biggest kernel for estimation. */
434 const DeviceKernel test_kernel = (kernel_features & KERNEL_FEATURE_NODE_RAYTRACE) ?
437
438 /* Launch kernel, using just 1 block appears sufficient to reserve memory for all
439 * multiprocessors. It would be good to do this in parallel for the multi GPU case
440 * still to make it faster. */
441 unique_ptr<DeviceQueue> queue = gpu_queue_create();
442
443 device_ptr d_path_index = 0;
444 device_ptr d_render_buffer = 0;
445 int d_work_size = 0;
446 DeviceKernelArguments args(&d_path_index, &d_render_buffer, &d_work_size);
447
448 queue->init_execution();
449 queue->enqueue(test_kernel, 1, args);
450 queue->synchronize();
451 }
452
453 {
454 HIPContextScope scope(this);
455 hipMemGetInfo(&free_after, &total);
456 }
457
458 LOG_INFO << "Local memory reserved " << string_human_readable_number(free_before - free_after)
459 << " bytes. (" << string_human_readable_size(free_before - free_after) << ")";
460
461# if 0
462 /* For testing mapped host memory, fill up device memory. */
463 const size_t keep_mb = 1024;
464
465 while (free_after > keep_mb * 1024 * 1024LL) {
466 hipDeviceptr_t tmp;
467 hip_assert(hipMalloc(&tmp, 10 * 1024 * 1024LL));
468 hipMemGetInfo(&free_after, &total);
469 }
470# endif
471}
472
473void HIPDevice::get_device_memory_info(size_t &total, size_t &free)
474{
475 HIPContextScope scope(this);
476
477 hipMemGetInfo(&free, &total);
478}
479
480bool HIPDevice::alloc_device(void *&device_pointer, const size_t size)
481{
482 HIPContextScope scope(this);
483
484 hipError_t mem_alloc_result = hipMalloc((hipDeviceptr_t *)&device_pointer, size);
485 return mem_alloc_result == hipSuccess;
486}
487
488void HIPDevice::free_device(void *device_pointer)
489{
490 HIPContextScope scope(this);
491
492 hip_assert(hipFree((hipDeviceptr_t)device_pointer));
493}
494
495bool HIPDevice::shared_alloc(void *&shared_pointer, const size_t size)
496{
497 HIPContextScope scope(this);
498
499 hipError_t mem_alloc_result = hipHostMalloc(
500 &shared_pointer, size, hipHostMallocMapped | hipHostMallocWriteCombined);
501
502 return mem_alloc_result == hipSuccess;
503}
504
505void HIPDevice::shared_free(void *shared_pointer, const size_t /*size*/)
506{
507 HIPContextScope scope(this);
508
509 hipHostFree(shared_pointer);
510}
511
512void *HIPDevice::shared_to_device_pointer(const void *shared_pointer)
513{
514 HIPContextScope scope(this);
515 void *device_pointer = nullptr;
516 hip_assert(
517 hipHostGetDevicePointer((hipDeviceptr_t *)&device_pointer, (void *)shared_pointer, 0));
518 return device_pointer;
519}
520
521void HIPDevice::copy_host_to_device(void *device_pointer, void *host_pointer, const size_t size)
522{
523 const HIPContextScope scope(this);
524
525 hip_assert(hipMemcpyHtoD((hipDeviceptr_t)device_pointer, host_pointer, size));
526}
527
528void HIPDevice::mem_alloc(device_memory &mem)
529{
530 if (mem.type == MEM_IMAGE_TEXTURE) {
531 assert(!"mem_alloc not supported for images.");
532 }
533 else if (mem.type == MEM_GLOBAL) {
534 assert(!"mem_alloc not supported for global memory.");
535 }
536 else {
537 generic_alloc(mem);
538 }
539}
540
541void HIPDevice::mem_copy_to(device_memory &mem)
542{
543 if (mem.type == MEM_GLOBAL) {
544 global_copy_to(mem);
545 }
546 else if (mem.type == MEM_IMAGE_TEXTURE) {
547 image_copy_to((device_image &)mem);
548 }
549 else {
550 if (!mem.device_pointer) {
551 generic_alloc(mem);
552 generic_copy_to(mem);
553 }
554 else if (mem.is_resident(this)) {
555 generic_copy_to(mem);
556 }
557 }
558}
559
560void HIPDevice::mem_move_to_host(device_memory &mem)
561{
562 if (mem.type == MEM_GLOBAL) {
563 global_free(mem);
564 global_alloc(mem);
565 }
566 else if (mem.type == MEM_IMAGE_TEXTURE) {
567 image_free((device_image &)mem);
568 image_alloc((device_image &)mem);
569 }
570 else {
571 assert(!"mem_move_to_host only supported for image and global memory");
572 }
573}
574
575void HIPDevice::mem_copy_from(
576 device_memory &mem, const size_t y, size_t w, const size_t h, size_t elem)
577{
578 if (mem.type == MEM_IMAGE_TEXTURE) {
579 assert(!"mem_copy_from not supported for images.");
580 }
581 else if (mem.host_pointer) {
582 const size_t size = elem * w * h;
583 const size_t offset = elem * y * w;
584
585 if (mem.device_pointer) {
586 const HIPContextScope scope(this);
587 hip_assert(hipMemcpyDtoH(
588 (char *)mem.host_pointer + offset, (hipDeviceptr_t)mem.device_pointer + offset, size));
589 }
590 else {
591 memset((char *)mem.host_pointer + offset, 0, size);
592 }
593 }
594}
595
596void HIPDevice::mem_zero(device_memory &mem)
597{
598 if (!mem.device_pointer) {
599 mem_alloc(mem);
600 }
601 if (!mem.device_pointer) {
602 return;
603 }
604
605 if (!(mem.is_shared(this) && mem.host_pointer == mem.shared_pointer)) {
606 const HIPContextScope scope(this);
607 hip_assert(hipMemsetD8((hipDeviceptr_t)mem.device_pointer, 0, mem.memory_size()));
608 }
609 else if (mem.host_pointer) {
610 memset(mem.host_pointer, 0, mem.memory_size());
611 }
612}
613
614void HIPDevice::mem_free(device_memory &mem)
615{
616 if (mem.type == MEM_GLOBAL) {
617 global_free(mem);
618 }
619 else if (mem.type == MEM_IMAGE_TEXTURE) {
620 image_free((device_image &)mem);
621 }
622 else {
623 generic_free(mem);
624 }
625}
626
627device_ptr HIPDevice::mem_alloc_sub_ptr(device_memory &mem, const size_t offset, size_t /*size*/)
628{
629 return (device_ptr)(((char *)mem.device_pointer) + mem.memory_elements_size(offset));
630}
631
632void HIPDevice::const_copy_to(const char *name, void *host, const size_t size)
633{
634 HIPContextScope scope(this);
635 hipDeviceptr_t mem;
636 size_t bytes;
637
638 hip_assert(hipModuleGetGlobal(&mem, &bytes, hipModule, "kernel_params"));
639 assert(bytes == sizeof(KernelParamsHIP));
640
641 /* Update data storage pointers in launch parameters. */
642# define KERNEL_DATA_ARRAY(data_type, data_name) \
643 if (strcmp(name, #data_name) == 0) { \
644 hip_assert(hipMemcpyHtoD(mem + offsetof(KernelParamsHIP, data_name), host, size)); \
645 return; \
646 }
647 KERNEL_DATA_ARRAY(KernelData, data)
648 KERNEL_DATA_ARRAY(IntegratorStateGPU, integrator_state)
649# include "kernel/data_arrays.h"
650# undef KERNEL_DATA_ARRAY
651}
652
653void HIPDevice::global_alloc(device_memory &mem)
654{
655 if (mem.is_resident(this)) {
656 generic_alloc(mem);
657 generic_copy_to(mem);
658 }
659
660 const_copy_to(mem.global_name(), &mem.device_pointer, sizeof(mem.device_pointer));
661}
662
663void HIPDevice::global_copy_to(device_memory &mem)
664{
665 if (!mem.device_pointer) {
666 generic_alloc(mem);
667 generic_copy_to(mem);
668 }
669 else if (mem.is_resident(this)) {
670 generic_copy_to(mem);
671 }
672
673 const_copy_to(mem.global_name(), &mem.device_pointer, sizeof(mem.device_pointer));
674}
675
676void HIPDevice::global_free(device_memory &mem)
677{
678 if (mem.is_resident(this) && mem.device_pointer) {
679 generic_free(mem);
680 }
681}
682
683static size_t tex_src_pitch(const device_image &mem)
684{
685 return mem.data_width * datatype_size(mem.data_type) * mem.data_elements;
686}
687
688static hip_Memcpy2D tex_2d_copy_param(const device_image &mem, const int pitch_alignment)
689{
690 /* 2D image using pitch aligned linear memory. */
691 const size_t src_pitch = tex_src_pitch(mem);
692 const size_t dst_pitch = align_up(src_pitch, pitch_alignment);
693
694 hip_Memcpy2D param;
695 memset(&param, 0, sizeof(param));
696 param.dstMemoryType = hipMemoryTypeDevice;
697 param.dstDevice = mem.device_pointer;
698 param.dstPitch = dst_pitch;
699 param.srcMemoryType = hipMemoryTypeHost;
700 param.srcHost = mem.host_pointer;
701 param.srcPitch = src_pitch;
702 param.WidthInBytes = param.srcPitch;
703 param.Height = mem.data_height;
704
705 return param;
706}
707
708void HIPDevice::image_alloc(device_image &mem)
709{
710 HIPContextScope scope(this);
711
712 hipTextureAddressMode address_mode = hipAddressModeWrap;
713 switch (mem.info.extension) {
714 case EXTENSION_REPEAT:
715 address_mode = hipAddressModeWrap;
716 break;
717 case EXTENSION_EXTEND:
718 address_mode = hipAddressModeClamp;
719 break;
720 case EXTENSION_CLIP:
721 address_mode = hipAddressModeBorder;
722 break;
723 case EXTENSION_MIRROR:
724 address_mode = hipAddressModeMirror;
725 break;
726 default:
727 assert(0);
728 break;
729 }
730
731 hipTextureFilterMode filter_mode;
733 filter_mode = hipFilterModePoint;
734 }
735 else {
736 filter_mode = hipFilterModeLinear;
737 }
738
739 /* Image Storage. */
740 hipArray_Format format;
741 switch (mem.data_type) {
742 case TYPE_UCHAR:
743 format = HIP_AD_FORMAT_UNSIGNED_INT8;
744 break;
745 case TYPE_UINT16:
746 format = HIP_AD_FORMAT_UNSIGNED_INT16;
747 break;
748 case TYPE_UINT:
749 format = HIP_AD_FORMAT_UNSIGNED_INT32;
750 break;
751 case TYPE_INT:
752 format = HIP_AD_FORMAT_SIGNED_INT32;
753 break;
754 case TYPE_FLOAT:
755 format = HIP_AD_FORMAT_FLOAT;
756 break;
757 case TYPE_HALF:
758 format = HIP_AD_FORMAT_HALF;
759 break;
760 default:
761 assert(0);
762 return;
763 }
764
765 Mem *cmem = nullptr;
766
767 if (!mem.is_resident(this)) {
768 thread_scoped_lock lock(device_mem_map_mutex);
769 cmem = &device_mem_map[&mem];
770 cmem->texobject = 0;
771 }
772 else if (mem.data_height > 0) {
773 /* 2D image, using pitch aligned linear memory. */
774 const size_t dst_pitch = align_up(tex_src_pitch(mem), pitch_alignment);
775 const size_t dst_size = dst_pitch * mem.data_height;
776
777 cmem = generic_alloc(mem, dst_size - mem.memory_size());
778 if (!cmem) {
779 return;
780 }
781
782 const hip_Memcpy2D param = tex_2d_copy_param(mem, pitch_alignment);
783 hip_assert(hipDrvMemcpy2DUnaligned(&param));
784 }
785 else {
786 /* 1D image, using linear memory. */
787 cmem = generic_alloc(mem);
788 if (!cmem) {
789 return;
790 }
791
792 hip_assert(hipMemcpyHtoD(mem.device_pointer, mem.host_pointer, mem.memory_size()));
793 }
794
795 /* Set Mapping and tag that we need to (re-)upload to device */
796 KernelImageInfo tex_info = mem.info;
797
798 if (!is_nanovdb_type(mem.info.data_type)) {
799 /* Bindless textures. */
800 hipResourceDesc resDesc;
801 memset(&resDesc, 0, sizeof(resDesc));
802
803 if (mem.data_height > 0) {
804 const size_t dst_pitch = align_up(tex_src_pitch(mem), pitch_alignment);
805
806 resDesc.resType = hipResourceTypePitch2D;
807 resDesc.res.pitch2D.devPtr = mem.device_pointer;
808 resDesc.res.pitch2D.format = format;
809 resDesc.res.pitch2D.numChannels = mem.data_elements;
810 resDesc.res.pitch2D.height = mem.data_height;
811 resDesc.res.pitch2D.width = mem.data_width;
812 resDesc.res.pitch2D.pitchInBytes = dst_pitch;
813 }
814 else {
815 resDesc.resType = hipResourceTypeLinear;
816 resDesc.res.linear.devPtr = mem.device_pointer;
817 resDesc.res.linear.format = format;
818 resDesc.res.linear.numChannels = mem.data_elements;
819 resDesc.res.linear.sizeInBytes = mem.device_size;
820 }
821
822 hipTextureDesc texDesc;
823 memset(&texDesc, 0, sizeof(texDesc));
824 texDesc.addressMode[0] = address_mode;
825 texDesc.addressMode[1] = address_mode;
826 texDesc.addressMode[2] = address_mode;
827 texDesc.filterMode = filter_mode;
828 texDesc.flags = HIP_TRSF_NORMALIZED_COORDINATES;
829
830 thread_scoped_lock lock(device_mem_map_mutex);
831 cmem = &device_mem_map[&mem];
832
833 if (hipTexObjectCreate(&cmem->texobject, &resDesc, &texDesc, nullptr) != hipSuccess) {
834 set_error(
835 "Failed to create image. Maximum GPU image size or available GPU memory was likely "
836 "exceeded.");
837 }
838
839 tex_info.data = (uint64_t)cmem->texobject;
840 }
841 else {
842 tex_info.data = (uint64_t)mem.device_pointer;
843 }
844
845 /* Update image info. */
846 image_info_update(mem.image_info_id, tex_info);
847}
848
849void HIPDevice::image_copy_to(device_image &mem)
850{
851 if (!mem.device_pointer) {
852 /* Not yet allocated on device. */
853 image_alloc(mem);
854 }
855 else if (!mem.is_resident(this)) {
856 /* Peering with another device, may still need to create image info and object. */
857 bool image_allocated = false;
858 {
859 thread_scoped_lock lock(image_info_mutex);
860 image_allocated = mem.image_info_id < image_info.size() &&
861 image_info[mem.image_info_id].data != 0;
862 }
863 if (!image_allocated) {
864 image_alloc(mem);
865 }
866 }
867 else {
868 /* Resident and fully allocated, only copy. */
869 if (mem.data_height > 0) {
870 HIPContextScope scope(this);
871 const hip_Memcpy2D param = tex_2d_copy_param(mem, pitch_alignment);
872 hip_assert(hipDrvMemcpy2DUnaligned(&param));
873 }
874 else {
875 generic_copy_to(mem);
876 }
877 }
878}
879
880void HIPDevice::image_free(device_image &mem)
881{
882 HIPContextScope scope(this);
883 thread_scoped_lock lock(device_mem_map_mutex);
884
885 /* Check if the memory was allocated for this device. */
886 auto it = device_mem_map.find(&mem);
887 if (it == device_mem_map.end()) {
888 return;
889 }
890
891 const Mem &cmem = it->second;
892
893 /* Always clear image info and texture object, regardless of residency. */
894 {
895 thread_scoped_lock lock(image_info_mutex);
896 image_info[mem.image_info_id] = KernelImageInfo();
897 }
898
899 if (cmem.texobject) {
900 /* Free bindless texture. */
901 hipTexObjectDestroy(cmem.texobject);
902 }
903
904 if (!mem.is_resident(this)) {
905 /* Do not free memory here, since it was allocated on a different device. */
906 device_mem_map.erase(device_mem_map.find(&mem));
907 }
908 else if (cmem.array) {
909 /* Free array. */
910 hipArrayDestroy(reinterpret_cast<hArray>(cmem.array));
911 stats.mem_free(mem.device_size);
912 mem.device_pointer = 0;
913 mem.device_size = 0;
914
915 device_mem_map.erase(device_mem_map.find(&mem));
916 }
917 else {
918 lock.unlock();
919 generic_free(mem);
920 }
921}
922
923unique_ptr<DeviceQueue> HIPDevice::gpu_queue_create()
924{
925 return make_unique<HIPDeviceQueue>(this);
926}
927
928bool HIPDevice::should_use_graphics_interop(const GraphicsInteropDevice &interop_device,
929 const bool log)
930{
931 if (headless) {
932 /* Avoid any call which might involve interaction with a graphics backend when we know that
933 * we don't have active graphics context. This avoids potential crash in the driver. */
934 return false;
935 }
936
937 HIPContextScope scope(this);
938
939 switch (interop_device.type) {
941 /* Disable graphics interop for now, because of driver bug in 21.40. See #92972.
942 * Also missing Vulkan support which is needed now. */
943 return false;
944
945 /* Check whether this device is part of OpenGL context.
946 *
947 * Using HIP device for graphics interoperability which is not part of the OpenGL context is
948 * possible, but from the empiric measurements with CUDA it can be considerably slower than
949 * using naive pixels copy. */
950 int num_all_devices = 0;
951 hip_assert(hipGetDeviceCount(&num_all_devices));
952
953 if (num_all_devices == 0) {
954 return false;
955 }
956
957 vector<hipDevice_t> gl_devices(num_all_devices);
958 uint num_gl_devices = 0;
959 hipGLGetDevices(&num_gl_devices, gl_devices.data(), num_all_devices, hipGLDeviceListAll);
960
961 bool found = false;
962 for (hipDevice_t gl_device : gl_devices) {
963 if (gl_device == hipDevice) {
964 found = true;
965 break;
966 }
967 }
968
969 if (log) {
970 if (found) {
971 LOG_INFO << "Graphics interop: found matching OpenGL device for HIP";
972 }
973 else {
974 LOG_INFO << "Graphics interop: no matching OpenGL device for HIP";
975 }
976 }
977
978 return found;
979 }
983 /* TODO: Implement Vulkan support. */
984 return false;
985 }
986
987 return false;
988}
989
990int HIPDevice::get_num_multiprocessors()
991{
992 return get_device_default_attribute(hipDeviceAttributeMultiprocessorCount, 0);
993}
994
995int HIPDevice::get_max_num_threads_per_multiprocessor()
996{
997 return get_device_default_attribute(hipDeviceAttributeMaxThreadsPerMultiProcessor, 0);
998}
999
1000bool HIPDevice::get_device_attribute(hipDeviceAttribute_t attribute, int *value)
1001{
1002 HIPContextScope scope(this);
1003
1004 return hipDeviceGetAttribute(value, attribute, hipDevice) == hipSuccess;
1005}
1006
1007int HIPDevice::get_device_default_attribute(hipDeviceAttribute_t attribute,
1008 const int default_value)
1009{
1010 int value = 0;
1011 if (!get_device_attribute(attribute, &value)) {
1012 return default_value;
1013 }
1014 return value;
1015}
1016
1018
1019#endif
volatile int lock
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition btDbvt.cpp:52
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
Definition btQuadWord.h:119
HIP hip
Definition debug.h:152
DeviceType type
virtual void set_error(const string &error)
DeviceInfo info
void mem_free(const size_t size)
Definition util/stats.h:24
KernelImageInfo info
bool is_resident(Device *sub_device) const
Definition memory.cpp:193
size_t memory_elements_size(const int elements)
const char * global_name() const
Definition memory.cpp:83
bool is_shared(Device *sub_device) const
Definition memory.cpp:198
@ MEM_IMAGE_TEXTURE
@ TYPE_UINT16
CCL_NAMESPACE_BEGIN struct Options options
#define KERNEL_DATA_ARRAY(type, name)
Definition data_arrays.h:8
#define KERNEL_FEATURE_NODE_RAYTRACE
#define CCL_NAMESPACE_END
@ DEVICE_OPTIX
@ DEVICE_HIP
#define assert(assertion)
#define attribute(slot)
#define log
@ BVH_LAYOUT_BVH2
DeviceKernel
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
#define LOG_INFO_IMPORTANT
Definition log.h:105
#define LOG_ERROR
Definition log.h:101
#define LOG_INFO
Definition log.h:106
string util_md5_string(const string &str)
Definition md5.cpp:386
unsigned int uint
Definition BKE_text.h:55
std::string format(const std::string &fmt, Args &&...args)
Definition diagnostic.hh:43
int BVHLayoutMask
Definition params.h:50
string path_cache_get(const string &sub)
Definition path.cpp:383
bool path_create_directories(const string &filepath)
Definition path.cpp:696
string path_get(const string &sub)
Definition path.cpp:368
string path_files_md5_hash(const string &dir)
Definition path.cpp:660
string path_join(const string &dir, const string &file)
Definition path.cpp:436
bool path_exists(const string &path)
Definition path.cpp:603
bool path_read_compressed_text(const string &path, string &text)
Definition path.cpp:803
string string_human_readable_size(size_t size)
Definition string.cpp:333
string to_string(const char *str)
Definition string.cpp:202
string string_human_readable_number(size_t num)
Definition string.cpp:352
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
Definition string.cpp:25
bool adaptive_compile
Definition debug.h:73
int system_cpu_bits()
Definition system.cpp:140
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
CCL_NAMESPACE_BEGIN double time_dt()
Definition time.cpp:31
uint64_t device_ptr
Definition types_base.h:44
ccl_device_inline size_t align_up(const size_t offset, const size_t alignment)
Definition types_base.h:48
@ INTERPOLATION_CLOSEST
Definition types_image.h:27
@ EXTENSION_REPEAT
Definition types_image.h:82
@ EXTENSION_CLIP
Definition types_image.h:86
@ EXTENSION_EXTEND
Definition types_image.h:84
@ EXTENSION_MIRROR
Definition types_image.h:88
ccl_device_inline bool is_nanovdb_type(int type)
Definition types_image.h:54