13# include <OSL/oslversion.h>
14# include <OpenImageIO/oiioversion.h>
41bool CUDADevice::have_precompiled_kernels()
43 string cubins_path =
path_get(
"lib");
52void CUDADevice::set_error(
const string &error)
57 LOG_ERROR <<
"Refer to the Cycles GPU rendering documentation for possible solutions:\n"
58 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
64 :
GPUDevice(info, stats, profiler, headless)
67 static_assert(
sizeof(texMemObject) ==
sizeof(
CUtexObject));
68 static_assert(
sizeof(arrayMemObject) ==
sizeof(CUarray));
78 need_image_info =
false;
83 CUresult
result = cuInit(0);
84 if (result != CUDA_SUCCESS) {
85 set_error(
string_printf(
"Failed to initialize CUDA runtime (%s)", cuewErrorString(result)));
90 result = cuDeviceGet(&cuDevice, cuDevId);
91 if (result != CUDA_SUCCESS) {
92 set_error(
string_printf(
"Failed to get CUDA device handle from ordinal (%s)",
93 cuewErrorString(result)));
101 cuda_assert(cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, cuDevice));
102 can_map_host = value != 0;
104 cuda_assert(cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_INTEGRATED, cuDevice));
105 integrated_gpu = value != 0;
107 cuda_assert(cuDeviceGetAttribute(
108 &pitch_alignment, CU_DEVICE_ATTRIBUTE_TEXTURE_PITCH_ALIGNMENT, cuDevice));
110 cuda_assert(cuDeviceGetAttribute(
111 &max_shared_mem_bytes, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, cuDevice));
118 unsigned int ctx_flags = 0;
119 cuda_assert(cuDevicePrimaryCtxGetState(cuDevice, &ctx_flags, &active));
123 ctx_flags |= CU_CTX_LMEM_RESIZE_TO_MAX;
124 result = cuDevicePrimaryCtxSetFlags(cuDevice, ctx_flags);
125 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PRIMARY_CONTEXT_ACTIVE) {
126 set_error(
string_printf(
"Failed to configure CUDA context (%s)", cuewErrorString(result)));
132 result = cuDevicePrimaryCtxRetain(&cuContext, cuDevice);
134 if (result != CUDA_SUCCESS) {
135 set_error(
string_printf(
"Failed to retain CUDA context (%s)", cuewErrorString(result)));
140 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
141 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
142 cuDevArchitecture = major * 100 + minor * 10;
145CUDADevice::~CUDADevice()
149 cuda_assert(cuModuleUnload(cuModule));
151 cuda_assert(cuDevicePrimaryCtxRelease(cuDevice));
154bool CUDADevice::support_device(
const uint64_t )
157 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
158 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
163 "CUDA backend requires compute capability 5.0 or up, but found %d.%d.", major, minor));
170bool CUDADevice::check_peer_access(
Device *peer_device)
172 if (peer_device ==
this) {
179 CUDADevice *
const peer_device_cuda =
static_cast<CUDADevice *
>(peer_device);
182 cuda_assert(cuDeviceCanAccessPeer(&can_access, cuDevice, peer_device_cuda->cuDevice));
183 if (can_access == 0) {
188 cuda_assert(cuDeviceGetP2PAttribute(&can_access,
189 CU_DEVICE_P2P_ATTRIBUTE_CUDA_ARRAY_ACCESS_SUPPORTED,
191 peer_device_cuda->cuDevice));
192 if (can_access == 0) {
198 const CUDAContextScope scope(
this);
199 CUresult
result = cuCtxEnablePeerAccess(peer_device_cuda->cuContext, 0);
200 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
201 set_error(
string_printf(
"Failed to enable peer access on CUDA context (%s)",
202 cuewErrorString(result)));
207 const CUDAContextScope scope(peer_device_cuda);
208 CUresult
result = cuCtxEnablePeerAccess(cuContext, 0);
209 if (result != CUDA_SUCCESS && result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
210 set_error(
string_printf(
"Failed to enable peer access on CUDA context (%s)",
211 cuewErrorString(result)));
219bool CUDADevice::use_adaptive_compilation()
227string CUDADevice::compile_kernel_get_common_cflags(
const uint64_t kernel_features)
230 const string source_path =
path_get(
"source");
231 const string include_path = source_path;
235 "--ptxas-options=\"-v\" "
240 include_path.c_str());
241 if (use_adaptive_compilation()) {
242 cflags +=
" -D__KERNEL_FEATURES__=" +
to_string(kernel_features);
244 const char *extra_cflags = getenv(
"CYCLES_CUDA_EXTRA_CFLAGS");
246 cflags += string(
" ") + string(extra_cflags);
250 cflags +=
" -DWITH_NANOVDB";
255 " -DOSL_LIBRARY_VERSION_CODE=%d -DOIIO_VERSION=%d", OSL_LIBRARY_VERSION_CODE, OIIO_VERSION);
258# ifdef WITH_CYCLES_DEBUG
259 cflags +=
" -DWITH_CYCLES_DEBUG";
265string CUDADevice::compile_kernel(
const string &common_cflags,
const char *name,
bool optix)
269 cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, cuDevId);
270 cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, cuDevId);
281 else if (!use_adaptive_compilation()) {
283 int cubin_minor = minor;
284 while (cubin_minor >= 0) {
286 string_printf(
"lib/%s_sm_%d%d.cubin.zst", name, major, cubin_minor));
287 LOG_INFO <<
"Testing for pre-compiled kernel " << cubin <<
".";
289 LOG_INFO <<
"Using precompiled kernel.";
297 int ptx_major = major, ptx_minor = minor;
298 while (ptx_major >= 5) {
300 string_printf(
"lib/%s_compute_%d%d.ptx.zst", name, ptx_major, ptx_minor));
301 LOG_INFO <<
"Testing for pre-compiled kernel " << ptx <<
".";
303 LOG_INFO <<
"Using precompiled kernel.";
318 string source_path =
path_get(
"source");
326 const char *
const kernel_ext = optix ?
"ptx" :
"cubin";
327 const char *
const kernel_arch = optix ?
"compute" :
"sm";
329 "cycles_%s_%s_%d%d_%s.%s", name, kernel_arch, major, minor, kernel_md5.c_str(), kernel_ext);
331 LOG_INFO <<
"Testing for locally compiled kernel " << cubin <<
".";
333 LOG_INFO <<
"Using locally compiled kernel.";
338 if (!use_adaptive_compilation() && have_precompiled_kernels()) {
341 string_printf(
"CUDA backend requires compute capability 5.0 or up, but found %d.%d. "
342 "Your GPU is not supported.",
348 string_printf(
"CUDA binary kernel for this graphics card compute "
349 "capability (%d.%d) not found.",
358 const char *
const nvcc = cuewCompilerPath();
359 if (nvcc ==
nullptr) {
361 "CUDA nvcc compiler not found. "
362 "Install CUDA toolkit in default location.");
366 const int nvcc_cuda_version = cuewCompilerVersion();
367 LOG_INFO <<
"Found nvcc " << nvcc <<
", CUDA version " << nvcc_cuda_version <<
".";
368 if (nvcc_cuda_version < 101) {
369 LOG_ERROR <<
"Unsupported CUDA version " << nvcc_cuda_version / 10 <<
"."
370 << nvcc_cuda_version % 10 <<
", you need CUDA 10.1 or newer";
373 if (!(nvcc_cuda_version >= 102 && nvcc_cuda_version < 130)) {
374 LOG_ERROR <<
"CUDA version " << nvcc_cuda_version / 10 <<
"." << nvcc_cuda_version % 10
375 <<
" detected, build may succeed but only CUDA 10.1 to 12 are officially supported.";
399 common_cflags.c_str());
401 LOG_INFO_IMPORTANT <<
"Compiling " << ((use_adaptive_compilation()) ?
"adaptive " :
"")
402 <<
"CUDA kernel ...";
406 command =
"call " + command;
408 if (system(command.c_str()) != 0) {
410 "Failed to execute compilation command, "
411 "see console for details.");
418 "CUDA kernel compilation failed, "
419 "see console for details.");
423 LOG_INFO_IMPORTANT <<
"Kernel compilation finished in " << std::fixed << std::setprecision(2)
424 <<
time_dt() - starttime <<
"s";
429bool CUDADevice::load_kernels(
const uint64_t kernel_features)
437 if (use_adaptive_compilation()) {
438 LOG_INFO <<
"Skipping CUDA kernel reload for adaptive compilation, not currently supported.";
444 if (cuContext ==
nullptr) {
449 if (!support_device(kernel_features)) {
454 const char *kernel_name =
"kernel";
455 string cflags = compile_kernel_get_common_cflags(kernel_features);
456 string cubin = compile_kernel(cflags, kernel_name);
462 CUDAContextScope scope(
this);
468 result = cuModuleLoadData(&cuModule, cubin_data.c_str());
471 result = CUDA_ERROR_FILE_NOT_FOUND;
474 if (result != CUDA_SUCCESS) {
476 "Failed to load CUDA kernel from '%s' (%s)", cubin.c_str(), cuewErrorString(result)));
479 if (result == CUDA_SUCCESS) {
481 reserve_local_memory(kernel_features);
484 return (result == CUDA_SUCCESS);
487void CUDADevice::reserve_local_memory(
const uint64_t kernel_features)
492 size_t total = 0, free_before = 0, free_after = 0;
495 CUDAContextScope scope(
this);
496 cuMemGetInfo(&free_before, &total);
508 CUDADeviceQueue queue(
this);
515 queue.init_execution();
516 queue.enqueue(test_kernel, 1, args);
521 CUDAContextScope scope(
this);
522 cuMemGetInfo(&free_after, &total);
530 const size_t keep_mb = 1024;
532 while (free_after > keep_mb * 1024 * 1024LL) {
534 cuda_assert(cuMemAlloc(&tmp, 10 * 1024 * 1024LL));
535 cuMemGetInfo(&free_after, &total);
559void CUDADevice::get_device_memory_info(
size_t &total,
size_t &free)
561 CUDAContextScope scope(
this);
563 cuMemGetInfo(&free, &total);
566bool CUDADevice::alloc_device(
void *&device_pointer,
const size_t size)
568 CUDAContextScope scope(
this);
570 CUresult mem_alloc_result = cuMemAlloc((CUdeviceptr *)&device_pointer,
size);
571 return mem_alloc_result == CUDA_SUCCESS;
574void CUDADevice::free_device(
void *device_pointer)
576 CUDAContextScope scope(
this);
578 cuda_assert(cuMemFree((CUdeviceptr)device_pointer));
581bool CUDADevice::shared_alloc(
void *&shared_pointer,
const size_t size)
583 const CUDAContextScope scope(
this);
589 if (!shared_pointer) {
592 const unsigned int flags = CU_MEMHOSTREGISTER_PORTABLE | CU_MEMHOSTREGISTER_DEVICEMAP;
593 if (cuMemHostRegister(shared_pointer,
size, flags) == CUDA_SUCCESS) {
597 shared_pointer =
nullptr;
600 const unsigned int flags = CU_MEMHOSTALLOC_PORTABLE | CU_MEMHOSTALLOC_DEVICEMAP;
601 CUresult mem_alloc_result = cuMemHostAlloc(&shared_pointer,
size, flags);
602 return mem_alloc_result == CUDA_SUCCESS;
606void CUDADevice::shared_free(
void *shared_pointer,
const size_t size)
608 const CUDAContextScope scope(
this);
611 cuMemHostUnregister(shared_pointer);
614 cuMemFreeHost(shared_pointer);
618void *CUDADevice::shared_to_device_pointer(
const void *shared_pointer)
620 const CUDAContextScope scope(
this);
622 void *device_pointer =
nullptr;
624 cuMemHostGetDevicePointer((CUdeviceptr *)&device_pointer, (
void *)shared_pointer, 0));
625 return device_pointer;
628void CUDADevice::copy_host_to_device(
void *device_pointer,
void *host_pointer,
const size_t size)
630 const CUDAContextScope scope(
this);
632 cuda_assert(cuMemcpyHtoD((CUdeviceptr)device_pointer, host_pointer,
size));
638 assert(!
"mem_alloc not supported for images.");
641 assert(!
"mem_alloc not supported for global memory.");
659 generic_copy_to(mem);
662 generic_copy_to(mem);
678 assert(!
"mem_move_to_host only supported for image and global memory");
682void CUDADevice::mem_copy_from(
683 device_memory &mem,
const size_t y,
size_t w,
const size_t h,
size_t elem)
686 assert(!
"mem_copy_from not supported for images.");
690 const size_t size = elem *
w * h;
691 const size_t offset = elem *
y *
w;
705 const CUDAContextScope scope(
this);
707 cuda_assert(cuMemcpyDtoH(
725 const CUDAContextScope scope(
this);
748void CUDADevice::const_copy_to(
const char *name,
void *host,
const size_t size)
750 CUDAContextScope scope(
this);
754 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule,
"kernel_params"));
757 if (strcmp(name,
"data") == 0) {
760 scene_max_shaders_ =
static_cast<const KernelData *
>(host)->max_shaders;
764# define KERNEL_DATA_ARRAY(data_type, data_name) \
765 if (strcmp(name, #data_name) == 0) { \
766 cuda_assert(cuMemcpyHtoD(mem + offsetof(KernelParamsCUDA, data_name), host, size)); \
771# include "kernel/data_arrays.h"
772# undef KERNEL_DATA_ARRAY
779 generic_copy_to(mem);
789 generic_copy_to(mem);
792 generic_copy_to(mem);
810static CUDA_MEMCPY2D tex_2d_copy_param(
const device_image &mem,
const int pitch_alignment)
813 const size_t src_pitch = tex_src_pitch(mem);
814 const size_t dst_pitch =
align_up(src_pitch, pitch_alignment);
817 memset(¶m, 0,
sizeof(param));
818 param.dstMemoryType = CU_MEMORYTYPE_DEVICE;
820 param.dstPitch = dst_pitch;
821 param.srcMemoryType = CU_MEMORYTYPE_HOST;
823 param.srcPitch = src_pitch;
824 param.WidthInBytes = param.srcPitch;
832 CUDAContextScope scope(
this);
834 CUaddress_mode address_mode = CU_TR_ADDRESS_MODE_WRAP;
837 address_mode = CU_TR_ADDRESS_MODE_WRAP;
840 address_mode = CU_TR_ADDRESS_MODE_CLAMP;
843 address_mode = CU_TR_ADDRESS_MODE_BORDER;
846 address_mode = CU_TR_ADDRESS_MODE_MIRROR;
853 CUfilter_mode filter_mode;
855 filter_mode = CU_TR_FILTER_MODE_POINT;
858 filter_mode = CU_TR_FILTER_MODE_LINEAR;
873 CUarray_format_enum
format;
876 format = CU_AD_FORMAT_UNSIGNED_INT8;
879 format = CU_AD_FORMAT_UNSIGNED_INT16;
882 format = CU_AD_FORMAT_FLOAT;
885 format = CU_AD_FORMAT_HALF;
896 cmem = &device_mem_map[&mem];
901 const size_t dst_pitch =
align_up(tex_src_pitch(mem), pitch_alignment);
902 const size_t dst_size = dst_pitch * mem.
data_height;
904 cmem = generic_alloc(mem, dst_size - mem.
memory_size());
910 const CUDA_MEMCPY2D param = tex_2d_copy_param(mem, pitch_alignment);
911 cuda_assert(cuMemcpy2DUnaligned(¶m));
916 cmem = generic_alloc(mem);
930 CUDA_RESOURCE_DESC resDesc;
931 memset(&resDesc, 0,
sizeof(resDesc));
934 const size_t dst_pitch =
align_up(tex_src_pitch(mem), pitch_alignment);
936 resDesc.resType = CU_RESOURCE_TYPE_PITCH2D;
938 resDesc.res.pitch2D.format =
format;
942 resDesc.res.pitch2D.pitchInBytes = dst_pitch;
945 resDesc.resType = CU_RESOURCE_TYPE_LINEAR;
947 resDesc.res.linear.format =
format;
952 CUDA_TEXTURE_DESC texDesc;
953 memset(&texDesc, 0,
sizeof(texDesc));
954 texDesc.addressMode[0] = address_mode;
955 texDesc.addressMode[1] = address_mode;
956 texDesc.addressMode[2] = address_mode;
957 texDesc.filterMode = filter_mode;
960 texDesc.flags = CU_TRSF_NORMALIZED_COORDINATES;
963 cmem = &device_mem_map[&mem];
965 cuda_assert(cuTexObjectCreate(&cmem->texobject, &resDesc, &texDesc,
nullptr));
985 bool image_allocated =
false;
991 if (!image_allocated) {
998 CUDAContextScope scope(
this);
999 const CUDA_MEMCPY2D param = tex_2d_copy_param(mem, pitch_alignment);
1000 cuda_assert(cuMemcpy2DUnaligned(¶m));
1003 generic_copy_to(mem);
1010 CUDAContextScope scope(
this);
1014 auto it = device_mem_map.find(&mem);
1015 if (it == device_mem_map.end()) {
1019 const Mem &cmem = it->second;
1027 if (cmem.texobject) {
1029 cuTexObjectDestroy(cmem.texobject);
1034 device_mem_map.erase(device_mem_map.find(&mem));
1036 else if (cmem.array) {
1038 cuArrayDestroy(
reinterpret_cast<CUarray
>(cmem.array));
1043 device_mem_map.erase(device_mem_map.find(&mem));
1053 return make_unique<CUDADeviceQueue>(
this);
1066 CUDAContextScope scope(
this);
1068 switch (interop_device.
type) {
1075 int num_all_devices = 0;
1076 cuda_assert(cuDeviceGetCount(&num_all_devices));
1078 if (num_all_devices == 0) {
1083 uint num_gl_devices = 0;
1084 cuGLGetDevices(&num_gl_devices, gl_devices.data(), num_all_devices, CU_GL_DEVICE_LIST_ALL);
1087 for (uint i = 0; i < num_gl_devices; ++i) {
1088 if (gl_devices[i] == cuDevice) {
1096 LOG_INFO <<
"Graphics interop: found matching OpenGL device for CUDA";
1099 LOG_INFO <<
"Graphics interop: no matching OpenGL device for CUDA";
1105 case ccl::GraphicsInteropDevice::VULKAN: {
1108 cuDeviceGetUuid(&uuid, cuDevice);
1109 const bool found = (
sizeof(uuid.bytes) == interop_device.
uuid.size() &&
1110 memcmp(uuid.bytes, interop_device.
uuid.data(),
sizeof(uuid.bytes)) == 0);
1114 LOG_INFO <<
"Graphics interop: found matching Vulkan device for CUDA";
1117 LOG_INFO <<
"Graphics interop: no matching Vulkan device for CUDA";
1120 LOG_INFO <<
"Graphics Interop: CUDA UUID "
1137int CUDADevice::get_num_multiprocessors()
1139 return get_device_default_attribute(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, 0);
1142int CUDADevice::get_max_num_threads_per_multiprocessor()
1144 return get_device_default_attribute(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, 0);
1147bool CUDADevice::get_device_attribute(CUdevice_attribute
attribute,
int *value)
1149 CUDAContextScope scope(
this);
1151 return cuDeviceGetAttribute(value,
attribute, cuDevice) == CUDA_SUCCESS;
1154int CUDADevice::get_device_default_attribute(CUdevice_attribute
attribute,
const int default_value)
1157 if (!get_device_attribute(
attribute, &value)) {
1158 return default_value;
void * util_page_aligned_malloc(const size_t size)
void util_page_aligned_free(void *ptr, const size_t size)
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
virtual void set_error(const string &error)
virtual GPUDevice::Mem * generic_alloc(device_memory &mem, const size_t pitch_padding=0)
void mem_free(const size_t size)
bool is_resident(Device *sub_device) const
size_t memory_elements_size(const int elements)
const char * global_name() const
bool is_shared(Device *sub_device) const
device_ptr device_pointer
#define KERNEL_DATA_ARRAY(type, name)
#define KERNEL_FEATURE_NODE_RAYTRACE
unsigned long long CUtexObject
#define CCL_NAMESPACE_END
#define assert(assertion)
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
#define LOG_INFO_IMPORTANT
string util_md5_string(const string &str)
std::string format(const std::string &fmt, Args &&...args)
string path_cache_get(const string &sub)
bool path_create_directories(const string &filepath)
string path_get(const string &sub)
string path_files_md5_hash(const string &dir)
string path_join(const string &dir, const string &file)
bool path_exists(const string &path)
bool path_read_compressed_text(const string &path, string &text)
string string_human_readable_size(size_t size)
string to_string(const char *str)
string string_hex(const uint8_t *data, const size_t size)
string string_human_readable_number(size_t num)
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
std::unique_lock< std::mutex > thread_scoped_lock
CCL_NAMESPACE_BEGIN double time_dt()
ccl_device_inline size_t align_up(const size_t offset, const size_t alignment)
ccl_device_inline bool is_nanovdb_type(int type)