35bool HIPDevice::have_precompiled_kernels()
37 string fatbins_path =
path_get(
"lib");
46void HIPDevice::set_error(
const string &error)
51 LOG_ERROR <<
"Refer to the Cycles GPU rendering documentation for possible solutions:\n"
52 "https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html\n";
58 :
GPUDevice(info, stats, profiler, headless)
61 static_assert(
sizeof(texMemObject) ==
sizeof(hipTextureObject_t));
62 static_assert(
sizeof(arrayMemObject) ==
sizeof(hArray));
72 need_image_info =
false;
77 hipError_t
result = hipInit(0);
78 if (result != hipSuccess) {
79 set_error(
string_printf(
"Failed to initialize HIP runtime (%s)", hipewErrorString(result)));
84 result = hipDeviceGet(&hipDevice, hipDevId);
85 if (result != hipSuccess) {
86 set_error(
string_printf(
"Failed to get HIP device handle from ordinal (%s)",
87 hipewErrorString(result)));
95 hip_assert(hipDeviceGetAttribute(&value, hipDeviceAttributeCanMapHostMemory, hipDevice));
96 can_map_host = value != 0;
99 hipDeviceGetAttribute(&pitch_alignment, hipDeviceAttributeTexturePitchAlignment, hipDevice));
101 unsigned int ctx_flags = hipDeviceLmemResizeToMax;
103 ctx_flags |= hipDeviceMapHost;
108 result = hipCtxCreate(&hipContext, ctx_flags, hipDevice);
110 if (result != hipSuccess) {
111 set_error(
string_printf(
"Failed to create HIP context (%s)", hipewErrorString(result)));
116 hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, hipDevId);
117 hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, hipDevId);
118 hipDevArchitecture = major * 100 + minor * 10;
121 hip_assert(hipRuntimeGetVersion(&hipRuntimeVersion));
124 hipCtxPopCurrent(
nullptr);
127HIPDevice::~HIPDevice()
131 hip_assert(hipModuleUnload(hipModule));
133 hip_assert(hipCtxDestroy(hipContext));
136bool HIPDevice::support_device(
const uint64_t )
138 if (hipSupportsDevice(hipDevId)) {
142 hipDeviceProp_t props;
143 hipGetDeviceProperties(&props, hipDevId);
145 set_error(
string_printf(
"HIP backend requires AMD RDNA graphics card or up, but found %s.",
150bool HIPDevice::check_peer_access(
Device *peer_device)
152 if (peer_device ==
this) {
159 HIPDevice *
const peer_device_hip =
static_cast<HIPDevice *
>(peer_device);
162 hip_assert(hipDeviceCanAccessPeer(&can_access, hipDevice, peer_device_hip->hipDevice));
163 if (can_access == 0) {
168 hip_assert(hipDeviceGetP2PAttribute(
169 &can_access, hipDevP2PAttrHipArrayAccessSupported, hipDevice, peer_device_hip->hipDevice));
170 if (can_access == 0) {
176 const HIPContextScope scope(
this);
177 hipError_t
result = hipCtxEnablePeerAccess(peer_device_hip->hipContext, 0);
178 if (result != hipSuccess) {
179 set_error(
string_printf(
"Failed to enable peer access on HIP context (%s)",
180 hipewErrorString(result)));
185 const HIPContextScope scope(peer_device_hip);
186 hipError_t
result = hipCtxEnablePeerAccess(hipContext, 0);
187 if (result != hipSuccess) {
188 set_error(
string_printf(
"Failed to enable peer access on HIP context (%s)",
189 hipewErrorString(result)));
197bool HIPDevice::use_adaptive_compilation()
205string HIPDevice::compile_kernel_get_common_cflags(
const uint64_t kernel_features)
208 const string source_path =
path_get(
"source");
209 const string include_path = source_path;
216 include_path.c_str());
217 if (use_adaptive_compilation()) {
218 cflags +=
" -D__KERNEL_FEATURES__=" +
to_string(kernel_features);
221 const char *extra_cflags = getenv(
"CYCLES_HIP_EXTRA_CFLAGS");
223 cflags += string(
" ") + string(extra_cflags);
227 cflags +=
" -DWITH_NANOVDB";
230# ifdef WITH_CYCLES_DEBUG
231 cflags +=
" -DWITH_CYCLES_DEBUG";
237string HIPDevice::compile_kernel(
const uint64_t kernel_features,
243 hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, hipDevId);
244 hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, hipDevId);
245 const std::string arch = hipDeviceArch(hipDevId);
248 if (!use_adaptive_compilation()) {
250 LOG_INFO <<
"Testing for pre-compiled kernel " << fatbin <<
".";
252 LOG_INFO <<
"Using precompiled kernel.";
258 string source_path =
path_get(
"source");
264 string common_cflags = compile_kernel_get_common_cflags(kernel_features);
267 const char *
const kernel_ext =
"genco";
268 std::string
options =
"-Wno-parentheses-equality -Wno-unused-value -ffast-math -std=c++17";
271 options.append(
" -save-temps");
273 if (major == 9 && minor == 0) {
277 options.append(
" --offload-arch=").append(arch);
279 const string include_path = source_path;
281 "cycles_%s_%s_%s", name, arch.c_str(), kernel_md5.c_str());
283 LOG_INFO <<
"Testing for locally compiled kernel " << fatbin <<
".";
285 LOG_INFO <<
"Using locally compiled kernel.";
290 if (!use_adaptive_compilation() && have_precompiled_kernels()) {
291 if (!hipSupportsDevice(hipDevId)) {
293 string_printf(
"HIP backend requires compute capability 10.1 or up, but found %d.%d. "
294 "Your GPU is not supported.",
300 string_printf(
"HIP binary kernel for this graphics card compute "
301 "capability (%d.%d) not found.",
310 const char *
const hipcc = hipewCompilerPath();
311 if (hipcc ==
nullptr) {
313 "HIP hipcc compiler not found. "
314 "Install HIP toolkit in default location.");
318 const int hipcc_hip_version = hipewCompilerVersion();
319 LOG_INFO <<
"Found hipcc " << hipcc <<
", HIP version " << hipcc_hip_version <<
".";
328 string command =
string_printf(
"%s %s -I \"%s\" --%s \"%s\" -o \"%s\" %s",
331 include_path.c_str(),
335 common_cflags.c_str());
337 LOG_INFO_IMPORTANT <<
"Compiling " << ((use_adaptive_compilation()) ?
"adaptive " :
"")
338 <<
"HIP kernel ... " << command;
341 command =
"call " + command;
343 if (system(command.c_str()) != 0) {
345 "Failed to execute compilation command, "
346 "see console for details.");
353 "HIP kernel compilation failed, "
354 "see console for details.");
358 LOG_INFO_IMPORTANT <<
"Kernel compilation finished in " << std::fixed << std::setprecision(2)
359 <<
time_dt() - starttime <<
"s";
364bool HIPDevice::load_kernels(
const uint64_t kernel_features)
371 if (use_adaptive_compilation()) {
372 LOG_INFO <<
"Skipping HIP kernel reload for adaptive compilation, not currently supported.";
378 if (hipContext ==
nullptr) {
383 if (!support_device(kernel_features)) {
388 const char *kernel_name =
"kernel";
389 string fatbin = compile_kernel(kernel_features, kernel_name);
390 if (fatbin.empty()) {
395 HIPContextScope scope(
this);
401 result = hipModuleLoadData(&hipModule, fatbin_data.c_str());
404 result = hipErrorFileNotFound;
407 if (result != hipSuccess) {
409 "Failed to load HIP kernel from '%s' (%s)", fatbin.c_str(), hipewErrorString(result)));
412 if (result == hipSuccess) {
413 kernels.load_all(
this, hipModule);
414 reserve_local_memory(kernel_features);
417 return (result == hipSuccess);
420void HIPDevice::reserve_local_memory(
const uint64_t kernel_features)
425 size_t total = 0, free_before = 0, free_after = 0;
428 HIPContextScope scope(
this);
429 hipMemGetInfo(&free_before, &total);
448 queue->init_execution();
449 queue->enqueue(test_kernel, 1, args);
450 queue->synchronize();
454 HIPContextScope scope(
this);
455 hipMemGetInfo(&free_after, &total);
463 const size_t keep_mb = 1024;
465 while (free_after > keep_mb * 1024 * 1024LL) {
467 hip_assert(hipMalloc(&tmp, 10 * 1024 * 1024LL));
468 hipMemGetInfo(&free_after, &total);
473void HIPDevice::get_device_memory_info(
size_t &total,
size_t &free)
475 HIPContextScope scope(
this);
477 hipMemGetInfo(&free, &total);
480bool HIPDevice::alloc_device(
void *&device_pointer,
const size_t size)
482 HIPContextScope scope(
this);
484 hipError_t mem_alloc_result = hipMalloc((hipDeviceptr_t *)&device_pointer,
size);
485 return mem_alloc_result == hipSuccess;
488void HIPDevice::free_device(
void *device_pointer)
490 HIPContextScope scope(
this);
492 hip_assert(hipFree((hipDeviceptr_t)device_pointer));
495bool HIPDevice::shared_alloc(
void *&shared_pointer,
const size_t size)
497 HIPContextScope scope(
this);
499 hipError_t mem_alloc_result = hipHostMalloc(
500 &shared_pointer,
size, hipHostMallocMapped | hipHostMallocWriteCombined);
502 return mem_alloc_result == hipSuccess;
505void HIPDevice::shared_free(
void *shared_pointer,
const size_t )
507 HIPContextScope scope(
this);
509 hipHostFree(shared_pointer);
512void *HIPDevice::shared_to_device_pointer(
const void *shared_pointer)
514 HIPContextScope scope(
this);
515 void *device_pointer =
nullptr;
517 hipHostGetDevicePointer((hipDeviceptr_t *)&device_pointer, (
void *)shared_pointer, 0));
518 return device_pointer;
521void HIPDevice::copy_host_to_device(
void *device_pointer,
void *host_pointer,
const size_t size)
523 const HIPContextScope scope(
this);
525 hip_assert(hipMemcpyHtoD((hipDeviceptr_t)device_pointer, host_pointer,
size));
531 assert(!
"mem_alloc not supported for images.");
534 assert(!
"mem_alloc not supported for global memory.");
552 generic_copy_to(mem);
555 generic_copy_to(mem);
571 assert(!
"mem_move_to_host only supported for image and global memory");
575void HIPDevice::mem_copy_from(
576 device_memory &mem,
const size_t y,
size_t w,
const size_t h,
size_t elem)
579 assert(!
"mem_copy_from not supported for images.");
582 const size_t size = elem *
w * h;
583 const size_t offset = elem *
y *
w;
586 const HIPContextScope scope(
this);
587 hip_assert(hipMemcpyDtoH(
606 const HIPContextScope scope(
this);
632void HIPDevice::const_copy_to(
const char *name,
void *host,
const size_t size)
634 HIPContextScope scope(
this);
638 hip_assert(hipModuleGetGlobal(&mem, &bytes, hipModule,
"kernel_params"));
642# define KERNEL_DATA_ARRAY(data_type, data_name) \
643 if (strcmp(name, #data_name) == 0) { \
644 hip_assert(hipMemcpyHtoD(mem + offsetof(KernelParamsHIP, data_name), host, size)); \
649# include "kernel/data_arrays.h"
650# undef KERNEL_DATA_ARRAY
657 generic_copy_to(mem);
667 generic_copy_to(mem);
670 generic_copy_to(mem);
688static hip_Memcpy2D tex_2d_copy_param(
const device_image &mem,
const int pitch_alignment)
691 const size_t src_pitch = tex_src_pitch(mem);
692 const size_t dst_pitch =
align_up(src_pitch, pitch_alignment);
695 memset(¶m, 0,
sizeof(param));
696 param.dstMemoryType = hipMemoryTypeDevice;
698 param.dstPitch = dst_pitch;
699 param.srcMemoryType = hipMemoryTypeHost;
701 param.srcPitch = src_pitch;
702 param.WidthInBytes = param.srcPitch;
710 HIPContextScope scope(
this);
712 hipTextureAddressMode address_mode = hipAddressModeWrap;
715 address_mode = hipAddressModeWrap;
718 address_mode = hipAddressModeClamp;
721 address_mode = hipAddressModeBorder;
724 address_mode = hipAddressModeMirror;
731 hipTextureFilterMode filter_mode;
733 filter_mode = hipFilterModePoint;
736 filter_mode = hipFilterModeLinear;
743 format = HIP_AD_FORMAT_UNSIGNED_INT8;
746 format = HIP_AD_FORMAT_UNSIGNED_INT16;
749 format = HIP_AD_FORMAT_UNSIGNED_INT32;
752 format = HIP_AD_FORMAT_SIGNED_INT32;
755 format = HIP_AD_FORMAT_FLOAT;
758 format = HIP_AD_FORMAT_HALF;
769 cmem = &device_mem_map[&mem];
774 const size_t dst_pitch =
align_up(tex_src_pitch(mem), pitch_alignment);
775 const size_t dst_size = dst_pitch * mem.
data_height;
777 cmem = generic_alloc(mem, dst_size - mem.
memory_size());
782 const hip_Memcpy2D param = tex_2d_copy_param(mem, pitch_alignment);
783 hip_assert(hipDrvMemcpy2DUnaligned(¶m));
787 cmem = generic_alloc(mem);
800 hipResourceDesc resDesc;
801 memset(&resDesc, 0,
sizeof(resDesc));
804 const size_t dst_pitch =
align_up(tex_src_pitch(mem), pitch_alignment);
806 resDesc.resType = hipResourceTypePitch2D;
808 resDesc.res.pitch2D.format =
format;
812 resDesc.res.pitch2D.pitchInBytes = dst_pitch;
815 resDesc.resType = hipResourceTypeLinear;
817 resDesc.res.linear.format =
format;
822 hipTextureDesc texDesc;
823 memset(&texDesc, 0,
sizeof(texDesc));
824 texDesc.addressMode[0] = address_mode;
825 texDesc.addressMode[1] = address_mode;
826 texDesc.addressMode[2] = address_mode;
827 texDesc.filterMode = filter_mode;
828 texDesc.flags = HIP_TRSF_NORMALIZED_COORDINATES;
831 cmem = &device_mem_map[&mem];
833 if (hipTexObjectCreate(&cmem->texobject, &resDesc, &texDesc,
nullptr) != hipSuccess) {
835 "Failed to create image. Maximum GPU image size or available GPU memory was likely "
857 bool image_allocated =
false;
863 if (!image_allocated) {
870 HIPContextScope scope(
this);
871 const hip_Memcpy2D param = tex_2d_copy_param(mem, pitch_alignment);
872 hip_assert(hipDrvMemcpy2DUnaligned(¶m));
875 generic_copy_to(mem);
882 HIPContextScope scope(
this);
886 auto it = device_mem_map.find(&mem);
887 if (it == device_mem_map.end()) {
891 const Mem &cmem = it->second;
899 if (cmem.texobject) {
901 hipTexObjectDestroy(cmem.texobject);
906 device_mem_map.erase(device_mem_map.find(&mem));
908 else if (cmem.array) {
910 hipArrayDestroy(
reinterpret_cast<hArray
>(cmem.array));
915 device_mem_map.erase(device_mem_map.find(&mem));
925 return make_unique<HIPDeviceQueue>(
this);
937 HIPContextScope scope(
this);
939 switch (interop_device.
type) {
950 int num_all_devices = 0;
951 hip_assert(hipGetDeviceCount(&num_all_devices));
953 if (num_all_devices == 0) {
958 uint num_gl_devices = 0;
959 hipGLGetDevices(&num_gl_devices, gl_devices.data(), num_all_devices, hipGLDeviceListAll);
962 for (hipDevice_t gl_device : gl_devices) {
963 if (gl_device == hipDevice) {
971 LOG_INFO <<
"Graphics interop: found matching OpenGL device for HIP";
974 LOG_INFO <<
"Graphics interop: no matching OpenGL device for HIP";
990int HIPDevice::get_num_multiprocessors()
992 return get_device_default_attribute(hipDeviceAttributeMultiprocessorCount, 0);
995int HIPDevice::get_max_num_threads_per_multiprocessor()
997 return get_device_default_attribute(hipDeviceAttributeMaxThreadsPerMultiProcessor, 0);
1000bool HIPDevice::get_device_attribute(hipDeviceAttribute_t
attribute,
int *value)
1002 HIPContextScope scope(
this);
1004 return hipDeviceGetAttribute(value,
attribute, hipDevice) == hipSuccess;
1007int HIPDevice::get_device_default_attribute(hipDeviceAttribute_t
attribute,
1008 const int default_value)
1011 if (!get_device_attribute(
attribute, &value)) {
1012 return default_value;
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
virtual void set_error(const string &error)
void mem_free(const size_t size)
bool is_resident(Device *sub_device) const
size_t memory_elements_size(const int elements)
const char * global_name() const
bool is_shared(Device *sub_device) const
device_ptr device_pointer
CCL_NAMESPACE_BEGIN struct Options options
#define KERNEL_DATA_ARRAY(type, name)
#define KERNEL_FEATURE_NODE_RAYTRACE
#define CCL_NAMESPACE_END
#define assert(assertion)
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE
@ DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE
#define LOG_INFO_IMPORTANT
string util_md5_string(const string &str)
std::string format(const std::string &fmt, Args &&...args)
string path_cache_get(const string &sub)
bool path_create_directories(const string &filepath)
string path_get(const string &sub)
string path_files_md5_hash(const string &dir)
string path_join(const string &dir, const string &file)
bool path_exists(const string &path)
bool path_read_compressed_text(const string &path, string &text)
string string_human_readable_size(size_t size)
string to_string(const char *str)
string string_human_readable_number(size_t num)
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
std::unique_lock< std::mutex > thread_scoped_lock
CCL_NAMESPACE_BEGIN double time_dt()
ccl_device_inline size_t align_up(const size_t offset, const size_t alignment)
ccl_device_inline bool is_nanovdb_type(int type)