Blender V5.3
cuda/device_impl.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#ifdef WITH_CUDA
6
7# include <cstdio>
8# include <cstdlib>
9# include <cstring>
10# include <iomanip>
11
12# ifdef WITH_OSL
13# include <OSL/oslversion.h>
14# include <OpenImageIO/oiioversion.h>
15# endif
16
18
19# include "util/debug.h"
20# include "util/log.h"
21# include "util/md5.h"
22# include "util/path.h"
23# include "util/string.h"
24# include "util/system.h"
25# include "util/time.h"
26# include "util/types.h"
27# include "util/types_image.h"
28
29# ifdef _WIN32
30# include "util/windows.h"
31# endif
32
34
36
38
39class CUDADevice;
40
41bool CUDADevice::have_precompiled_kernels()
42{
43 string cubins_path = path_get("lib");
44 return path_exists(cubins_path);
45}
46
47BVHLayoutMask CUDADevice::get_bvh_layout_mask(const uint64_t /*kernel_features*/) const
48{
49 return BVH_LAYOUT_BVH2;
50}
51
52void CUDADevice::set_error(const string &error)
53{
54 Device::set_error(error);
55
56 if (first_error) {
57 LOG_ERROR << "Refer to the Cycles GPU rendering documentation for possible solutions:\n"
58 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
59 first_error = false;
60 }
61}
62
63CUDADevice::CUDADevice(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
64 : GPUDevice(info, stats, profiler, headless)
65{
66 /* Verify that base class types can be used with specific backend types */
67 static_assert(sizeof(texMemObject) == sizeof(CUtexObject));
68 static_assert(sizeof(arrayMemObject) == sizeof(CUarray));
69
70 first_error = true;
71
72 cuDevId = info.num;
73 cuDevice = 0;
74 cuContext = nullptr;
75
76 cuModule = nullptr;
77
78 need_image_info = false;
79
80 pitch_alignment = 0;
81
82 /* Initialize CUDA. */
83 CUresult result = cuInit(0);
84 if (result != CUDA_SUCCESS) {
85 set_error(string_printf("Failed to initialize CUDA runtime (%s)", cuewErrorString(result)));
86 return;
87 }
88
89 /* Setup device and context. */
90 result = cuDeviceGet(&cuDevice, cuDevId);
91 if (result != CUDA_SUCCESS) {
92 set_error(string_printf("Failed to get CUDA device handle from ordinal (%s)",
93 cuewErrorString(result)));
94 return;
95 }
96
97 /* CU_CTX_MAP_HOST for mapping host memory when out of device memory.
98 * CU_CTX_LMEM_RESIZE_TO_MAX for reserving local memory ahead of render,
99 * so we can predict which memory to map to host. */
100 int value;
101 cuda_assert(cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, cuDevice));
102 can_map_host = value != 0;
103
104 cuda_assert(cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_INTEGRATED, cuDevice));
105 integrated_gpu = value != 0;
106
107 cuda_assert(cuDeviceGetAttribute(
108 &pitch_alignment, CU_DEVICE_ATTRIBUTE_TEXTURE_PITCH_ALIGNMENT, cuDevice));
109
110 cuda_assert(cuDeviceGetAttribute(
111 &max_shared_mem_bytes, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, cuDevice));
112
113 if (can_map_host) {
114 init_host_memory();
115 }
116
117 int active = 0;
118 unsigned int ctx_flags = 0;
119 cuda_assert(cuDevicePrimaryCtxGetState(cuDevice, &ctx_flags, &active));
120
121 /* Configure primary context only once. */
122 if (active == 0) {
123 ctx_flags |= CU_CTX_LMEM_RESIZE_TO_MAX;
124 result = cuDevicePrimaryCtxSetFlags(cuDevice, ctx_flags);
125 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PRIMARY_CONTEXT_ACTIVE) {
126 set_error(string_printf("Failed to configure CUDA context (%s)", cuewErrorString(result)));
127 return;
128 }
129 }
130
131 /* Create context. */
132 result = cuDevicePrimaryCtxRetain(&cuContext, cuDevice);
133
134 if (result != CUDA_SUCCESS) {
135 set_error(string_printf("Failed to retain CUDA context (%s)", cuewErrorString(result)));
136 return;
137 }
138
139 int major, minor;
140 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
141 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
142 cuDevArchitecture = major * 100 + minor * 10;
143}
144
145CUDADevice::~CUDADevice()
146{
147 image_info.free();
148 if (cuModule) {
149 cuda_assert(cuModuleUnload(cuModule));
150 }
151 cuda_assert(cuDevicePrimaryCtxRelease(cuDevice));
152}
153
154bool CUDADevice::support_device(const uint64_t /*kernel_features*/)
155{
156 int major, minor;
157 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
158 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
159
160 /* We only support sm_50 and above */
161 if (major < 5) {
162 set_error(string_printf(
163 "CUDA backend requires compute capability 5.0 or up, but found %d.%d.", major, minor));
164 return false;
165 }
166
167 return true;
168}
169
170bool CUDADevice::check_peer_access(Device *peer_device)
171{
172 if (peer_device == this) {
173 return false;
174 }
175 if (peer_device->info.type != DEVICE_CUDA && peer_device->info.type != DEVICE_OPTIX) {
176 return false;
177 }
178
179 CUDADevice *const peer_device_cuda = static_cast<CUDADevice *>(peer_device);
180
181 int can_access = 0;
182 cuda_assert(cuDeviceCanAccessPeer(&can_access, cuDevice, peer_device_cuda->cuDevice));
183 if (can_access == 0) {
184 return false;
185 }
186
187 // Ensure array access over the link is possible as well (for 3D images)
188 cuda_assert(cuDeviceGetP2PAttribute(&can_access,
189 CU_DEVICE_P2P_ATTRIBUTE_CUDA_ARRAY_ACCESS_SUPPORTED,
190 cuDevice,
191 peer_device_cuda->cuDevice));
192 if (can_access == 0) {
193 return false;
194 }
195
196 // Enable peer access in both directions
197 {
198 const CUDAContextScope scope(this);
199 CUresult result = cuCtxEnablePeerAccess(peer_device_cuda->cuContext, 0);
200 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
201 set_error(string_printf("Failed to enable peer access on CUDA context (%s)",
202 cuewErrorString(result)));
203 return false;
204 }
205 }
206 {
207 const CUDAContextScope scope(peer_device_cuda);
208 CUresult result = cuCtxEnablePeerAccess(cuContext, 0);
209 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
210 set_error(string_printf("Failed to enable peer access on CUDA context (%s)",
211 cuewErrorString(result)));
212 return false;
213 }
214 }
215
216 return true;
217}
218
219bool CUDADevice::use_adaptive_compilation()
220{
222}
223
224/* Common NVCC flags which stays the same regardless of shading model,
225 * kernel sources md5 and only depends on compiler or compilation settings.
226 */
227string CUDADevice::compile_kernel_get_common_cflags(const uint64_t kernel_features)
228{
229 const int machine = system_cpu_bits();
230 const string source_path = path_get("source");
231 const string include_path = source_path;
232 string cflags = string_printf(
233 "-std=c++17 "
234 "-m%d "
235 "--ptxas-options=\"-v\" "
236 "--use_fast_math "
237 "-DNVCC "
238 "-I\"%s\"",
239 machine,
240 include_path.c_str());
241 if (use_adaptive_compilation()) {
242 cflags += " -D__KERNEL_FEATURES__=" + to_string(kernel_features);
243 }
244 const char *extra_cflags = getenv("CYCLES_CUDA_EXTRA_CFLAGS");
245 if (extra_cflags) {
246 cflags += string(" ") + string(extra_cflags);
247 }
248
249# ifdef WITH_NANOVDB
250 cflags += " -DWITH_NANOVDB";
251# endif
252
253# ifdef WITH_OSL
254 cflags += string_printf(
255 " -DOSL_LIBRARY_VERSION_CODE=%d -DOIIO_VERSION=%d", OSL_LIBRARY_VERSION_CODE, OIIO_VERSION);
256# endif
257
258# ifdef WITH_CYCLES_DEBUG
259 cflags += " -DWITH_CYCLES_DEBUG";
260# endif
261
262 return cflags;
263}
264
265string CUDADevice::compile_kernel(const string &common_cflags, const char *name, bool optix)
266{
267 /* Compute kernel name. */
268 int major, minor;
269 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
270 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
271
272 if (optix) {
273 /* CUDA 13 introduced PTX verification for compute_90+, which is not compatible with OptiX
274 * device intrinsics, so avoid triggering it by targeting a lower version. */
275 if (major >= 9) {
276 major = 8;
277 minor = 9;
278 }
279 }
280 /* Attempt to use kernel provided with Blender. */
281 else if (!use_adaptive_compilation()) {
282 /* Binaries within a major version are compatible, so find the closest one. */
283 int cubin_minor = minor;
284 while (cubin_minor >= 0) {
285 const string cubin = path_get(
286 string_printf("lib/%s_sm_%d%d.cubin.zst", name, major, cubin_minor));
287 LOG_INFO << "Testing for pre-compiled kernel " << cubin << ".";
288 if (path_exists(cubin)) {
289 LOG_INFO << "Using precompiled kernel.";
290 return cubin;
291 }
292
293 cubin_minor--;
294 }
295
296 /* The driver can JIT-compile PTX generated for older generations, so find the closest one. */
297 int ptx_major = major, ptx_minor = minor;
298 while (ptx_major >= 5) {
299 const string ptx = path_get(
300 string_printf("lib/%s_compute_%d%d.ptx.zst", name, ptx_major, ptx_minor));
301 LOG_INFO << "Testing for pre-compiled kernel " << ptx << ".";
302 if (path_exists(ptx)) {
303 LOG_INFO << "Using precompiled kernel.";
304 return ptx;
305 }
306
307 if (ptx_minor > 0) {
308 ptx_minor--;
309 }
310 else {
311 ptx_major--;
312 ptx_minor = 9;
313 }
314 }
315 }
316
317 /* Try to use locally compiled kernel. */
318 string source_path = path_get("source");
319 const string source_md5 = path_files_md5_hash(source_path);
320
321 /* We include cflags into md5 so changing cuda toolkit or changing other
322 * compiler command line arguments makes sure cubin gets re-built.
323 */
324 const string kernel_md5 = util_md5_string(source_md5 + common_cflags);
325
326 const char *const kernel_ext = optix ? "ptx" : "cubin";
327 const char *const kernel_arch = optix ? "compute" : "sm";
328 const string cubin_file = string_printf(
329 "cycles_%s_%s_%d%d_%s.%s", name, kernel_arch, major, minor, kernel_md5.c_str(), kernel_ext);
330 const string cubin = path_cache_get(path_join("kernels", cubin_file));
331 LOG_INFO << "Testing for locally compiled kernel " << cubin << ".";
332 if (path_exists(cubin)) {
333 LOG_INFO << "Using locally compiled kernel.";
334 return cubin;
335 }
336
337# ifdef _WIN32
338 if (!use_adaptive_compilation() && have_precompiled_kernels()) {
339 if (major < 5) {
340 set_error(
341 string_printf("CUDA backend requires compute capability 5.0 or up, but found %d.%d. "
342 "Your GPU is not supported.",
343 major,
344 minor));
345 }
346 else {
347 set_error(
348 string_printf("CUDA binary kernel for this graphics card compute "
349 "capability (%d.%d) not found.",
350 major,
351 minor));
352 }
353 return string();
354 }
355# endif
356
357 /* Compile. */
358 const char *const nvcc = cuewCompilerPath();
359 if (nvcc == nullptr) {
360 set_error(
361 "CUDA nvcc compiler not found. "
362 "Install CUDA toolkit in default location.");
363 return string();
364 }
365
366 const int nvcc_cuda_version = cuewCompilerVersion();
367 LOG_INFO << "Found nvcc " << nvcc << ", CUDA version " << nvcc_cuda_version << ".";
368 if (nvcc_cuda_version < 101) {
369 LOG_ERROR << "Unsupported CUDA version " << nvcc_cuda_version / 10 << "."
370 << nvcc_cuda_version % 10 << ", you need CUDA 10.1 or newer";
371 return string();
372 }
373 if (!(nvcc_cuda_version >= 102 && nvcc_cuda_version < 130)) {
374 LOG_ERROR << "CUDA version " << nvcc_cuda_version / 10 << "." << nvcc_cuda_version % 10
375 << " detected, build may succeed but only CUDA 10.1 to 12 are officially supported.";
376 }
377
378 double starttime = time_dt();
379
381
382 source_path = path_join(
383 path_join(source_path, "kernel"),
384 path_join("device", path_join(optix ? "optix" : "cuda", string_printf("%s.cu", name))));
385
386 string command = string_printf(
387 "\"%s\" "
388 "-arch=%s_%d%d "
389 "--%s \"%s\" "
390 "-o \"%s\" "
391 "%s",
392 nvcc,
393 kernel_arch,
394 major,
395 minor,
396 kernel_ext,
397 source_path.c_str(),
398 cubin.c_str(),
399 common_cflags.c_str());
400
401 LOG_INFO_IMPORTANT << "Compiling " << ((use_adaptive_compilation()) ? "adaptive " : "")
402 << "CUDA kernel ...";
403 LOG_INFO_IMPORTANT << command;
404
405# ifdef _WIN32
406 command = "call " + command;
407# endif
408 if (system(command.c_str()) != 0) {
409 set_error(
410 "Failed to execute compilation command, "
411 "see console for details.");
412 return string();
413 }
414
415 /* Verify if compilation succeeded */
416 if (!path_exists(cubin)) {
417 set_error(
418 "CUDA kernel compilation failed, "
419 "see console for details.");
420 return string();
421 }
422
423 LOG_INFO_IMPORTANT << "Kernel compilation finished in " << std::fixed << std::setprecision(2)
424 << time_dt() - starttime << "s";
425
426 return cubin;
427}
428
429bool CUDADevice::load_kernels(const uint64_t kernel_features)
430{
431 /* TODO(sergey): Support kernels re-load for CUDA devices adaptive compile.
432 *
433 * Currently re-loading kernel will invalidate memory pointers,
434 * causing problems in cuCtxSynchronize.
435 */
436 if (cuModule) {
437 if (use_adaptive_compilation()) {
438 LOG_INFO << "Skipping CUDA kernel reload for adaptive compilation, not currently supported.";
439 }
440 return true;
441 }
442
443 /* check if cuda init succeeded */
444 if (cuContext == nullptr) {
445 return false;
446 }
447
448 /* check if GPU is supported */
449 if (!support_device(kernel_features)) {
450 return false;
451 }
452
453 /* get kernel */
454 const char *kernel_name = "kernel";
455 string cflags = compile_kernel_get_common_cflags(kernel_features);
456 string cubin = compile_kernel(cflags, kernel_name);
457 if (cubin.empty()) {
458 return false;
459 }
460
461 /* open module */
462 CUDAContextScope scope(this);
463
464 string cubin_data;
465 CUresult result;
466
467 if (path_read_compressed_text(cubin, cubin_data)) {
468 result = cuModuleLoadData(&cuModule, cubin_data.c_str());
469 }
470 else {
471 result = CUDA_ERROR_FILE_NOT_FOUND;
472 }
473
474 if (result != CUDA_SUCCESS) {
475 set_error(string_printf(
476 "Failed to load CUDA kernel from '%s' (%s)", cubin.c_str(), cuewErrorString(result)));
477 }
478
479 if (result == CUDA_SUCCESS) {
480 kernels.load(this);
481 reserve_local_memory(kernel_features);
482 }
483
484 return (result == CUDA_SUCCESS);
485}
486
487void CUDADevice::reserve_local_memory(const uint64_t kernel_features)
488{
489 /* Together with CU_CTX_LMEM_RESIZE_TO_MAX, this reserves local memory
490 * needed for kernel launches, so that we can reliably figure out when
491 * to allocate scene data in mapped host memory. */
492 size_t total = 0, free_before = 0, free_after = 0;
493
494 {
495 CUDAContextScope scope(this);
496 cuMemGetInfo(&free_before, &total);
497 }
498
499 {
500 /* Use the biggest kernel for estimation. */
501 const DeviceKernel test_kernel = (kernel_features & KERNEL_FEATURE_NODE_RAYTRACE) ?
504
505 /* Launch kernel, using just 1 block appears sufficient to reserve memory for all
506 * multiprocessors. It would be good to do this in parallel for the multi GPU case
507 * still to make it faster. */
508 CUDADeviceQueue queue(this);
509
510 device_ptr d_path_index = 0;
511 device_ptr d_render_buffer = 0;
512 int d_work_size = 0;
513 DeviceKernelArguments args(&d_path_index, &d_render_buffer, &d_work_size);
514
515 queue.init_execution();
516 queue.enqueue(test_kernel, 1, args);
517 queue.synchronize();
518 }
519
520 {
521 CUDAContextScope scope(this);
522 cuMemGetInfo(&free_after, &total);
523 }
524
525 LOG_INFO << "Local memory reserved " << string_human_readable_number(free_before - free_after)
526 << " bytes. (" << string_human_readable_size(free_before - free_after) << ")";
527
528# if 0
529 /* For testing mapped host memory, fill up device memory. */
530 const size_t keep_mb = 1024;
531
532 while (free_after > keep_mb * 1024 * 1024LL) {
533 CUdeviceptr tmp;
534 cuda_assert(cuMemAlloc(&tmp, 10 * 1024 * 1024LL));
535 cuMemGetInfo(&free_after, &total);
536 }
537# endif
538}
539
540GPUDevice::Mem *CUDADevice::generic_alloc(device_memory &mem, const size_t pitch_padding)
541{
542 /* Force frequently copied allocations into shared memory on integrated GPUs. */
543 if (integrated_gpu && mem.type == MEM_READ_WRITE && mem.host_pointer) {
544 mem.move_to_host = true;
545 Mem *const cmem = GPUDevice::generic_alloc(mem, pitch_padding);
546 mem.move_to_host = false;
547
548 if (cmem && mem.shared_pointer && mem.host_pointer != mem.shared_pointer) {
549 memcpy(mem.shared_pointer, mem.host_pointer, mem.memory_size());
550 host_free(mem.type, mem.host_pointer, mem.memory_size());
552 }
553 return cmem;
554 }
555
556 return GPUDevice::generic_alloc(mem, pitch_padding);
557}
558
559void CUDADevice::get_device_memory_info(size_t &total, size_t &free)
560{
561 CUDAContextScope scope(this);
562
563 cuMemGetInfo(&free, &total);
564}
565
566bool CUDADevice::alloc_device(void *&device_pointer, const size_t size)
567{
568 CUDAContextScope scope(this);
569
570 CUresult mem_alloc_result = cuMemAlloc((CUdeviceptr *)&device_pointer, size);
571 return mem_alloc_result == CUDA_SUCCESS;
572}
573
574void CUDADevice::free_device(void *device_pointer)
575{
576 CUDAContextScope scope(this);
577
578 cuda_assert(cuMemFree((CUdeviceptr)device_pointer));
579}
580
581bool CUDADevice::shared_alloc(void *&shared_pointer, const size_t size)
582{
583 const CUDAContextScope scope(this);
584
585# if 1
586 /* Register memory aligned to large page boundaries, since on integrated GPUs that is currently
587 * the most optimal path for performance. */
588 shared_pointer = util_page_aligned_malloc(size);
589 if (!shared_pointer) {
590 return false;
591 }
592 const unsigned int flags = CU_MEMHOSTREGISTER_PORTABLE | CU_MEMHOSTREGISTER_DEVICEMAP;
593 if (cuMemHostRegister(shared_pointer, size, flags) == CUDA_SUCCESS) {
594 return true;
595 }
596 util_page_aligned_free(shared_pointer, size);
597 shared_pointer = nullptr;
598 return false;
599# else
600 const unsigned int flags = CU_MEMHOSTALLOC_PORTABLE | CU_MEMHOSTALLOC_DEVICEMAP;
601 CUresult mem_alloc_result = cuMemHostAlloc(&shared_pointer, size, flags);
602 return mem_alloc_result == CUDA_SUCCESS;
603# endif
604}
605
606void CUDADevice::shared_free(void *shared_pointer, const size_t size)
607{
608 const CUDAContextScope scope(this);
609
610# if 1
611 cuMemHostUnregister(shared_pointer);
612 util_page_aligned_free(shared_pointer, size);
613# else
614 cuMemFreeHost(shared_pointer);
615# endif
616}
617
618void *CUDADevice::shared_to_device_pointer(const void *shared_pointer)
619{
620 const CUDAContextScope scope(this);
621
622 void *device_pointer = nullptr;
623 cuda_assert(
624 cuMemHostGetDevicePointer((CUdeviceptr *)&device_pointer, (void *)shared_pointer, 0));
625 return device_pointer;
626}
627
628void CUDADevice::copy_host_to_device(void *device_pointer, void *host_pointer, const size_t size)
629{
630 const CUDAContextScope scope(this);
631
632 cuda_assert(cuMemcpyHtoD((CUdeviceptr)device_pointer, host_pointer, size));
633}
634
635void CUDADevice::mem_alloc(device_memory &mem)
636{
637 if (mem.type == MEM_IMAGE_TEXTURE) {
638 assert(!"mem_alloc not supported for images.");
639 }
640 else if (mem.type == MEM_GLOBAL) {
641 assert(!"mem_alloc not supported for global memory.");
642 }
643 else {
644 generic_alloc(mem);
645 }
646}
647
648void CUDADevice::mem_copy_to(device_memory &mem)
649{
650 if (mem.type == MEM_GLOBAL) {
651 global_copy_to(mem);
652 }
653 else if (mem.type == MEM_IMAGE_TEXTURE) {
654 image_copy_to((device_image &)mem);
655 }
656 else {
657 if (!mem.device_pointer) {
658 generic_alloc(mem);
659 generic_copy_to(mem);
660 }
661 else if (mem.is_resident(this)) {
662 generic_copy_to(mem);
663 }
664 }
665}
666
667void CUDADevice::mem_move_to_host(device_memory &mem)
668{
669 if (mem.type == MEM_GLOBAL) {
670 global_free(mem);
671 global_alloc(mem);
672 }
673 else if (mem.type == MEM_IMAGE_TEXTURE) {
674 image_free((device_image &)mem);
675 image_alloc((device_image &)mem);
676 }
677 else {
678 assert(!"mem_move_to_host only supported for image and global memory");
679 }
680}
681
682void CUDADevice::mem_copy_from(
683 device_memory &mem, const size_t y, size_t w, const size_t h, size_t elem)
684{
685 if (mem.type == MEM_IMAGE_TEXTURE) {
686 assert(!"mem_copy_from not supported for images.");
687 return;
688 }
689
690 const size_t size = elem * w * h;
691 const size_t offset = elem * y * w;
692
693 if (!mem.host_pointer) {
694 return;
695 }
696 if (!mem.device_pointer) {
697 memset((char *)mem.host_pointer + offset, 0, size);
698 return;
699 }
700
701 if (mem.is_shared(this) && mem.host_pointer == mem.shared_pointer) {
702 return;
703 }
704
705 const CUDAContextScope scope(this);
706
707 cuda_assert(cuMemcpyDtoH(
708 (char *)mem.host_pointer + offset, (CUdeviceptr)mem.device_pointer + offset, size));
709}
710
711void CUDADevice::mem_zero(device_memory &mem)
712{
713 if (!mem.device_pointer) {
714 mem_alloc(mem);
715 }
716 if (!mem.device_pointer) {
717 return;
718 }
719
720 if (mem.is_shared(this) && mem.host_pointer == mem.shared_pointer) {
721 memset(mem.host_pointer, 0, mem.memory_size());
722 return;
723 }
724
725 const CUDAContextScope scope(this);
726
727 cuda_assert(cuMemsetD8((CUdeviceptr)mem.device_pointer, 0, mem.memory_size()));
728}
729
730void CUDADevice::mem_free(device_memory &mem)
731{
732 if (mem.type == MEM_GLOBAL) {
733 global_free(mem);
734 }
735 else if (mem.type == MEM_IMAGE_TEXTURE) {
736 image_free((device_image &)mem);
737 }
738 else {
739 generic_free(mem);
740 }
741}
742
743device_ptr CUDADevice::mem_alloc_sub_ptr(device_memory &mem, const size_t offset, size_t /*size*/)
744{
745 return (device_ptr)(((char *)mem.device_pointer) + mem.memory_elements_size(offset));
746}
747
748void CUDADevice::const_copy_to(const char *name, void *host, const size_t size)
749{
750 CUDAContextScope scope(this);
751 CUdeviceptr mem;
752 size_t bytes;
753
754 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule, "kernel_params"));
755 assert(bytes == sizeof(KernelParamsCUDA));
756
757 if (strcmp(name, "data") == 0) {
758 /* We need this value for shared memory size when launching integrator_sort_bucket_pass
759 * and integrator_sort_write_pass kernels. */
760 scene_max_shaders_ = static_cast<const KernelData *>(host)->max_shaders;
761 }
762
763 /* Update data storage pointers in launch parameters. */
764# define KERNEL_DATA_ARRAY(data_type, data_name) \
765 if (strcmp(name, #data_name) == 0) { \
766 cuda_assert(cuMemcpyHtoD(mem + offsetof(KernelParamsCUDA, data_name), host, size)); \
767 return; \
768 }
769 KERNEL_DATA_ARRAY(KernelData, data)
770 KERNEL_DATA_ARRAY(IntegratorStateGPU, integrator_state)
771# include "kernel/data_arrays.h"
772# undef KERNEL_DATA_ARRAY
773}
774
775void CUDADevice::global_alloc(device_memory &mem)
776{
777 if (mem.is_resident(this)) {
778 generic_alloc(mem);
779 generic_copy_to(mem);
780 }
781
782 const_copy_to(mem.global_name(), &mem.device_pointer, sizeof(mem.device_pointer));
783}
784
785void CUDADevice::global_copy_to(device_memory &mem)
786{
787 if (!mem.device_pointer) {
788 generic_alloc(mem);
789 generic_copy_to(mem);
790 }
791 else if (mem.is_resident(this)) {
792 generic_copy_to(mem);
793 }
794
795 const_copy_to(mem.global_name(), &mem.device_pointer, sizeof(mem.device_pointer));
796}
797
798void CUDADevice::global_free(device_memory &mem)
799{
800 if (mem.is_resident(this) && mem.device_pointer) {
801 generic_free(mem);
802 }
803}
804
805static size_t tex_src_pitch(const device_image &mem)
806{
807 return mem.data_width * datatype_size(mem.data_type) * mem.data_elements;
808}
809
810static CUDA_MEMCPY2D tex_2d_copy_param(const device_image &mem, const int pitch_alignment)
811{
812 /* 2D image using pitch aligned linear memory. */
813 const size_t src_pitch = tex_src_pitch(mem);
814 const size_t dst_pitch = align_up(src_pitch, pitch_alignment);
815
816 CUDA_MEMCPY2D param;
817 memset(&param, 0, sizeof(param));
818 param.dstMemoryType = CU_MEMORYTYPE_DEVICE;
819 param.dstDevice = mem.device_pointer;
820 param.dstPitch = dst_pitch;
821 param.srcMemoryType = CU_MEMORYTYPE_HOST;
822 param.srcHost = mem.host_pointer;
823 param.srcPitch = src_pitch;
824 param.WidthInBytes = param.srcPitch;
825 param.Height = mem.data_height;
826
827 return param;
828}
829
830void CUDADevice::image_alloc(device_image &mem)
831{
832 CUDAContextScope scope(this);
833
834 CUaddress_mode address_mode = CU_TR_ADDRESS_MODE_WRAP;
835 switch (mem.info.extension) {
836 case EXTENSION_REPEAT:
837 address_mode = CU_TR_ADDRESS_MODE_WRAP;
838 break;
839 case EXTENSION_EXTEND:
840 address_mode = CU_TR_ADDRESS_MODE_CLAMP;
841 break;
842 case EXTENSION_CLIP:
843 address_mode = CU_TR_ADDRESS_MODE_BORDER;
844 break;
845 case EXTENSION_MIRROR:
846 address_mode = CU_TR_ADDRESS_MODE_MIRROR;
847 break;
848 default:
849 assert(0);
850 break;
851 }
852
853 CUfilter_mode filter_mode;
855 filter_mode = CU_TR_FILTER_MODE_POINT;
856 }
857 else {
858 filter_mode = CU_TR_FILTER_MODE_LINEAR;
859 }
860
861 /* Image Texture Storage
862 *
863 * Cycles expects to read all image data as normalized float values in
864 * kernel/device/gpu/image.h. But storing all data as floats would be very inefficient due to the
865 * huge size of float images. So in the code below, we define different texture types including
866 * integer types, with the aim of using CUDA's default promotion behavior of integer data to
867 * floating point data in the range [0, 1], as noted in the CUDA documentation on
868 * cuTexObjectCreate API Call.
869 *
870 * Note that 32-bit integers are not supported by this promotion behavior and cannot be used
871 * with Cycles's current implementation in kernel/device/gpu/image.h.
872 */
873 CUarray_format_enum format;
874 switch (mem.data_type) {
875 case TYPE_UCHAR:
876 format = CU_AD_FORMAT_UNSIGNED_INT8;
877 break;
878 case TYPE_UINT16:
879 format = CU_AD_FORMAT_UNSIGNED_INT16;
880 break;
881 case TYPE_FLOAT:
882 format = CU_AD_FORMAT_FLOAT;
883 break;
884 case TYPE_HALF:
885 format = CU_AD_FORMAT_HALF;
886 break;
887 default:
888 assert(0);
889 return;
890 }
891
892 Mem *cmem = nullptr;
893
894 if (!mem.is_resident(this)) {
895 thread_scoped_lock lock(device_mem_map_mutex);
896 cmem = &device_mem_map[&mem];
897 cmem->texobject = 0;
898 }
899 else if (mem.data_height > 0) {
900 /* 2D image, using pitch aligned linear memory. */
901 const size_t dst_pitch = align_up(tex_src_pitch(mem), pitch_alignment);
902 const size_t dst_size = dst_pitch * mem.data_height;
903
904 cmem = generic_alloc(mem, dst_size - mem.memory_size());
905 if (!cmem) {
906 return;
907 }
908
909 if (!(mem.is_shared(this) && mem.host_pointer == mem.shared_pointer)) {
910 const CUDA_MEMCPY2D param = tex_2d_copy_param(mem, pitch_alignment);
911 cuda_assert(cuMemcpy2DUnaligned(&param));
912 }
913 }
914 else {
915 /* 1D image, using linear memory. */
916 cmem = generic_alloc(mem);
917 if (!cmem) {
918 return;
919 }
920
921 if (!(mem.is_shared(this) && mem.host_pointer == mem.shared_pointer)) {
922 cuda_assert(cuMemcpyHtoD(mem.device_pointer, mem.host_pointer, mem.memory_size()));
923 }
924 }
925
926 /* Set Mapping and tag that we need to (re-)upload to device */
927 KernelImageInfo tex_info = mem.info;
928
929 if (!is_nanovdb_type(mem.info.data_type)) {
930 CUDA_RESOURCE_DESC resDesc;
931 memset(&resDesc, 0, sizeof(resDesc));
932
933 if (mem.data_height > 0) {
934 const size_t dst_pitch = align_up(tex_src_pitch(mem), pitch_alignment);
935
936 resDesc.resType = CU_RESOURCE_TYPE_PITCH2D;
937 resDesc.res.pitch2D.devPtr = mem.device_pointer;
938 resDesc.res.pitch2D.format = format;
939 resDesc.res.pitch2D.numChannels = mem.data_elements;
940 resDesc.res.pitch2D.height = mem.data_height;
941 resDesc.res.pitch2D.width = mem.data_width;
942 resDesc.res.pitch2D.pitchInBytes = dst_pitch;
943 }
944 else {
945 resDesc.resType = CU_RESOURCE_TYPE_LINEAR;
946 resDesc.res.linear.devPtr = mem.device_pointer;
947 resDesc.res.linear.format = format;
948 resDesc.res.linear.numChannels = mem.data_elements;
949 resDesc.res.linear.sizeInBytes = mem.device_size;
950 }
951
952 CUDA_TEXTURE_DESC texDesc;
953 memset(&texDesc, 0, sizeof(texDesc));
954 texDesc.addressMode[0] = address_mode;
955 texDesc.addressMode[1] = address_mode;
956 texDesc.addressMode[2] = address_mode;
957 texDesc.filterMode = filter_mode;
958 /* CUDA's flag CU_TRSF_READ_AS_INTEGER is intentionally not used and it is
959 * significant, see above an explanation about how Blender treat images. */
960 texDesc.flags = CU_TRSF_NORMALIZED_COORDINATES;
961
962 thread_scoped_lock lock(device_mem_map_mutex);
963 cmem = &device_mem_map[&mem];
964
965 cuda_assert(cuTexObjectCreate(&cmem->texobject, &resDesc, &texDesc, nullptr));
966
967 tex_info.data = (uint64_t)cmem->texobject;
968 }
969 else {
970 tex_info.data = (uint64_t)mem.device_pointer;
971 }
972
973 /* Update image info. */
974 image_info_update(mem.image_info_id, tex_info);
975}
976
977void CUDADevice::image_copy_to(device_image &mem)
978{
979 if (!mem.device_pointer) {
980 /* Not yet allocated on device. */
981 image_alloc(mem);
982 }
983 else if (!mem.is_resident(this)) {
984 /* Peering with another device, may still need to create image info and object. */
985 bool image_allocated = false;
986 {
987 thread_scoped_lock lock(image_info_mutex);
988 image_allocated = mem.image_info_id < image_info.size() &&
989 image_info[mem.image_info_id].data != 0;
990 }
991 if (!image_allocated) {
992 image_alloc(mem);
993 }
994 }
995 else if (!(mem.is_shared(this) && mem.host_pointer == mem.shared_pointer)) {
996 /* Resident and fully allocated, only copy. */
997 if (mem.data_height > 0) {
998 CUDAContextScope scope(this);
999 const CUDA_MEMCPY2D param = tex_2d_copy_param(mem, pitch_alignment);
1000 cuda_assert(cuMemcpy2DUnaligned(&param));
1001 }
1002 else {
1003 generic_copy_to(mem);
1004 }
1005 }
1006}
1007
1008void CUDADevice::image_free(device_image &mem)
1009{
1010 CUDAContextScope scope(this);
1011 thread_scoped_lock lock(device_mem_map_mutex);
1012
1013 /* Check if the memory was allocated for this device. */
1014 auto it = device_mem_map.find(&mem);
1015 if (it == device_mem_map.end()) {
1016 return;
1017 }
1018
1019 const Mem &cmem = it->second;
1020
1021 /* Always clear image info and image object, regardless of residency. */
1022 {
1023 thread_scoped_lock lock(image_info_mutex);
1024 image_info[mem.image_info_id] = KernelImageInfo();
1025 }
1026
1027 if (cmem.texobject) {
1028 /* Free bindless texture. */
1029 cuTexObjectDestroy(cmem.texobject);
1030 }
1031
1032 if (!mem.is_resident(this)) {
1033 /* Do not free memory here, since it was allocated on a different device. */
1034 device_mem_map.erase(device_mem_map.find(&mem));
1035 }
1036 else if (cmem.array) {
1037 /* Free array. */
1038 cuArrayDestroy(reinterpret_cast<CUarray>(cmem.array));
1039 stats.mem_free(mem.device_size);
1040 mem.device_pointer = 0;
1041 mem.device_size = 0;
1042
1043 device_mem_map.erase(device_mem_map.find(&mem));
1044 }
1045 else {
1046 lock.unlock();
1047 generic_free(mem);
1048 }
1049}
1050
1051unique_ptr<DeviceQueue> CUDADevice::gpu_queue_create()
1052{
1053 return make_unique<CUDADeviceQueue>(this);
1054}
1055
1056bool CUDADevice::should_use_graphics_interop(const GraphicsInteropDevice &interop_device,
1057 const bool log)
1058{
1059 if (headless) {
1060 /* Avoid any call which might involve interaction with a graphics backend when we know that
1061 * we don't have active graphics context. This avoid crash on certain platforms when calling
1062 * cuGLGetDevices(). */
1063 return false;
1064 }
1065
1066 CUDAContextScope scope(this);
1067
1068 switch (interop_device.type) {
1070 /* Check whether this device is part of OpenGL context.
1071 *
1072 * Using CUDA device for graphics interoperability which is not part of the OpenGL context is
1073 * possible, but from the empiric measurements it can be considerably slower than using naive
1074 * pixels copy. */
1075 int num_all_devices = 0;
1076 cuda_assert(cuDeviceGetCount(&num_all_devices));
1077
1078 if (num_all_devices == 0) {
1079 return false;
1080 }
1081
1082 vector<CUdevice> gl_devices(num_all_devices);
1083 uint num_gl_devices = 0;
1084 cuGLGetDevices(&num_gl_devices, gl_devices.data(), num_all_devices, CU_GL_DEVICE_LIST_ALL);
1085
1086 bool found = false;
1087 for (uint i = 0; i < num_gl_devices; ++i) {
1088 if (gl_devices[i] == cuDevice) {
1089 found = true;
1090 break;
1091 }
1092 }
1093
1094 if (log) {
1095 if (found) {
1096 LOG_INFO << "Graphics interop: found matching OpenGL device for CUDA";
1097 }
1098 else {
1099 LOG_INFO << "Graphics interop: no matching OpenGL device for CUDA";
1100 }
1101 }
1102
1103 return found;
1104 }
1105 case ccl::GraphicsInteropDevice::VULKAN: {
1106 /* Only do interop with matching device UUID. */
1107 CUuuid uuid = {};
1108 cuDeviceGetUuid(&uuid, cuDevice);
1109 const bool found = (sizeof(uuid.bytes) == interop_device.uuid.size() &&
1110 memcmp(uuid.bytes, interop_device.uuid.data(), sizeof(uuid.bytes)) == 0);
1111
1112 if (log) {
1113 if (found) {
1114 LOG_INFO << "Graphics interop: found matching Vulkan device for CUDA";
1115 }
1116 else {
1117 LOG_INFO << "Graphics interop: no matching Vulkan device for CUDA";
1118 }
1119
1120 LOG_INFO << "Graphics Interop: CUDA UUID "
1121 << string_hex(reinterpret_cast<uint8_t *>(uuid.bytes), sizeof(uuid.bytes))
1122 << ", Vulkan UUID "
1123 << string_hex(interop_device.uuid.data(), interop_device.uuid.size());
1124 }
1125
1126 return found;
1127 }
1130 return false;
1131 }
1132 }
1133
1134 return false;
1135}
1136
1137int CUDADevice::get_num_multiprocessors()
1138{
1139 return get_device_default_attribute(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, 0);
1140}
1141
1142int CUDADevice::get_max_num_threads_per_multiprocessor()
1143{
1144 return get_device_default_attribute(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, 0);
1145}
1146
1147bool CUDADevice::get_device_attribute(CUdevice_attribute attribute, int *value)
1148{
1149 CUDAContextScope scope(this);
1150
1151 return cuDeviceGetAttribute(value, attribute, cuDevice) == CUDA_SUCCESS;
1152}
1153
1154int CUDADevice::get_device_default_attribute(CUdevice_attribute attribute, const int default_value)
1155{
1156 int value = 0;
1157 if (!get_device_attribute(attribute, &value)) {
1158 return default_value;
1159 }
1160 return value;
1161}
1162
1164
1165#endif
void * util_page_aligned_malloc(const size_t size)
void util_page_aligned_free(void *ptr, const size_t size)
volatile int lock
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition btDbvt.cpp:52
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
Definition btQuadWord.h:119
CUDA cuda
Definition debug.h:146
DeviceType type
virtual void set_error(const string &error)
DeviceInfo info
virtual GPUDevice::Mem * generic_alloc(device_memory &mem, const size_t pitch_padding=0)
void mem_free(const size_t size)
Definition util/stats.h:24
KernelImageInfo info
bool is_resident(Device *sub_device) const
Definition memory.cpp:193
size_t memory_elements_size(const int elements)
const char * global_name() const
Definition memory.cpp:83
bool is_shared(Device *sub_device) const
Definition memory.cpp:198
@ MEM_IMAGE_TEXTURE
@ MEM_READ_WRITE
@ TYPE_UINT16
#define KERNEL_DATA_ARRAY(type, name)
Definition data_arrays.h:8
#define KERNEL_FEATURE_NODE_RAYTRACE
unsigned long long CUtexObject
#define CCL_NAMESPACE_END
@ DEVICE_CUDA
@ DEVICE_OPTIX
#define assert(assertion)
#define attribute(slot)
#define log
@ BVH_LAYOUT_BVH2
DeviceKernel
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
#define LOG_INFO_IMPORTANT
Definition log.h:105
#define LOG_ERROR
Definition log.h:101
#define LOG_INFO
Definition log.h:106
string util_md5_string(const string &str)
Definition md5.cpp:386
unsigned int uint
Definition BKE_text.h:55
unsigned char uint8_t
Definition BKE_text.h:79
std::string format(const std::string &fmt, Args &&...args)
Definition diagnostic.hh:43
int BVHLayoutMask
Definition params.h:50
string path_cache_get(const string &sub)
Definition path.cpp:383
bool path_create_directories(const string &filepath)
Definition path.cpp:696
string path_get(const string &sub)
Definition path.cpp:368
string path_files_md5_hash(const string &dir)
Definition path.cpp:660
string path_join(const string &dir, const string &file)
Definition path.cpp:436
bool path_exists(const string &path)
Definition path.cpp:603
bool path_read_compressed_text(const string &path, string &text)
Definition path.cpp:803
string string_human_readable_size(size_t size)
Definition string.cpp:333
string to_string(const char *str)
Definition string.cpp:202
string string_hex(const uint8_t *data, const size_t size)
Definition string.cpp:193
string string_human_readable_number(size_t num)
Definition string.cpp:352
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
Definition string.cpp:25
bool adaptive_compile
Definition debug.h:59
int system_cpu_bits()
Definition system.cpp:140
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
CCL_NAMESPACE_BEGIN double time_dt()
Definition time.cpp:31
uint64_t device_ptr
Definition types_base.h:44
ccl_device_inline size_t align_up(const size_t offset, const size_t alignment)
Definition types_base.h:48
@ INTERPOLATION_CLOSEST
Definition types_image.h:27
@ EXTENSION_REPEAT
Definition types_image.h:82
@ EXTENSION_CLIP
Definition types_image.h:86
@ EXTENSION_EXTEND
Definition types_image.h:84
@ EXTENSION_MIRROR
Definition types_image.h:88
ccl_device_inline bool is_nanovdb_type(int type)
Definition types_image.h:54