Blender V5.3
hiprt/device_impl.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2023 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#ifdef WITH_HIPRT
6
8
9# include <hiprt/hiprt.h>
10# include <iomanip>
11
12# include "device/hip/util.h"
14
15# include "util/log.h"
16# include "util/md5.h"
17# include "util/path.h"
18# include "util/progress.h"
19# include "util/string.h"
20# include "util/time.h"
21# include "util/types.h"
22# include "util/vector.h"
23
24# if defined(_WIN32)
25# include "util/windows.h"
26# elif defined(__linux__)
27# include <dlfcn.h>
28# endif
29
30# include "bvh/hiprt.h"
31
32# include "scene/hair.h"
33# include "scene/mesh.h"
34# include "scene/object.h"
35# include "scene/pointcloud.h"
36
38
39static void get_hiprt_transform(float matrix[][4], Transform &tfm)
40{
41 int row = 0;
42 int col = 0;
43 matrix[row][col++] = tfm.x.x;
44 matrix[row][col++] = tfm.x.y;
45 matrix[row][col++] = tfm.x.z;
46 matrix[row][col++] = tfm.x.w;
47 row++;
48 col = 0;
49 matrix[row][col++] = tfm.y.x;
50 matrix[row][col++] = tfm.y.y;
51 matrix[row][col++] = tfm.y.z;
52 matrix[row][col++] = tfm.y.w;
53 row++;
54 col = 0;
55 matrix[row][col++] = tfm.z.x;
56 matrix[row][col++] = tfm.z.y;
57 matrix[row][col++] = tfm.z.z;
58 matrix[row][col++] = tfm.z.w;
59}
60
61bool HIPRTDevice::is_supported()
62{
63# if defined(__linux__)
64 static bool is_initialized = false;
65 static bool is_supported = false;
66
67 if (is_initialized) {
68 return is_supported;
69 }
70 is_initialized = true;
71
72 /* The current version of HIP-RT requires libamdhip64.so which Fedora puts in a separate package
73 * than libamdhip64.so.6 as required by HIP. For now check for the existence of this. In the
74 * future update we'll make HIP-RT consistent, and this code can be removed. */
75 const vector<const char *> hip_paths = {
76 "libamdhip64.so",
77 "/opt/rocm/lib/libamdhip64.so",
78 "/opt/rocm/hip/lib/libamdhip64.so",
79 };
80
81 LOG_INFO << "Checking for libamdhip64.so";
82
83 for (const char *hip_path : hip_paths) {
84 void *hip_lib = dlopen(hip_path, RTLD_LAZY);
85 if (hip_lib) {
86 LOG_DEBUG << "Found libamdhip64.so at: " << hip_path;
87 is_supported = true;
88 dlclose(hip_lib);
89 break;
90 }
91 }
92
93 if (!is_supported) {
94 LOG_INFO << "libamdhip64.so not found, HIP-RT will be disabled";
95 }
96
97 return is_supported;
98# else
99 return true;
100# endif
101}
102
103BVHLayoutMask HIPRTDevice::get_bvh_layout_mask(const uint64_t /*kernel_features*/) const
104{
105 return BVH_LAYOUT_HIPRT;
106}
107
108HIPRTDevice::HIPRTDevice(const DeviceInfo &info,
109 Stats &stats,
110 Profiler &profiler,
111 const bool headless)
112 : HIPDevice(info, stats, profiler, headless),
113 hiprt_context(nullptr),
114 hiprt_module_(nullptr),
115 scene(nullptr),
116 functions_table(nullptr),
117 scratch_buffer_size(0),
118 scratch_buffer(this, "scratch_buffer", MEM_DEVICE_ONLY),
119 prim_visibility(this, "prim_visibility", MEM_GLOBAL),
120 instance_transform_matrix(this, "instance_transform_matrix", MEM_READ_ONLY),
121 transform_headers(this, "transform_headers", MEM_READ_ONLY),
122 user_instance_id(this, "user_instance_id", MEM_GLOBAL),
123 hiprt_blas_ptr(this, "hiprt_blas_ptr", MEM_READ_WRITE),
124 blas_ptr(this, "blas_ptr", MEM_GLOBAL)
125{
126 HIPContextScope scope(this);
127 global_stack_buffer = {0};
128 hiprtContextCreationInput hiprt_context_input = {nullptr};
129 hiprt_context_input.ctxt = hipContext;
130 hiprt_context_input.device = hipDevice;
131 hiprt_context_input.deviceType = hiprtDeviceAMD;
132 hiprtError rt_result = hiprtCreateContext(HIPRT_API_VERSION, hiprt_context_input, hiprt_context);
133
134 if (rt_result != hiprtSuccess) {
135 set_error("Failed to create HIPRT context");
136 return;
137 }
138
139 rt_result = hiprtCreateFuncTable(
140 hiprt_context, Max_Primitive_Type, Max_Intersect_Filter_Function, functions_table);
141
142 if (rt_result != hiprtSuccess) {
143 set_error("Failed to create HIPRT Function Table");
144 return;
145 }
146
148 hiprtSetLogLevel(hiprt_context, hiprtLogLevelInfo | hiprtLogLevelWarn | hiprtLogLevelError);
149 }
150 else {
151 hiprtSetLogLevel(hiprt_context, hiprtLogLevelNone);
152 }
153}
154
155HIPRTDevice::~HIPRTDevice()
156{
157 HIPContextScope scope(this);
158 free_bvh_memory_delayed();
159 user_instance_id.free();
160 prim_visibility.free();
161 hiprt_blas_ptr.free();
162 blas_ptr.free();
163 instance_transform_matrix.free();
164 transform_headers.free();
165
166 hiprtDestroyGlobalStackBuffer(hiprt_context, global_stack_buffer);
167 hiprtDestroyFuncTable(hiprt_context, functions_table);
168 hiprtDestroyScene(hiprt_context, scene);
169
170 if (hiprt_module_) {
171 hip_assert(hipModuleUnload(hiprt_module_));
172 }
173
174 hiprtDestroyContext(hiprt_context);
175}
176
177unique_ptr<DeviceQueue> HIPRTDevice::gpu_queue_create()
178{
179 return make_unique<HIPRTDeviceQueue>(this);
180}
181
182string HIPRTDevice::compile_kernel_get_common_cflags(const uint64_t kernel_features)
183{
184 string cflags = HIPDevice::compile_kernel_get_common_cflags(kernel_features);
185
186 cflags += " -D __KERNEL_HIPRT__ ";
187
188 return cflags;
189}
190
191string HIPRTDevice::compile_kernel(const uint64_t kernel_features,
192 const char *name,
193 const char *base)
194{
195 int major, minor;
196 hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, hipDevId);
197 hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, hipDevId);
198 const std::string arch = hipDeviceArch(hipDevId);
199
200 if (!use_adaptive_compilation()) {
201 const string fatbin = path_get(string_printf("lib/%s_rt_%s.hipfb.zst", name, arch.c_str()));
202 LOG_INFO << "Testing for pre-compiled kernel " << fatbin << ".";
203 if (path_exists(fatbin)) {
204 LOG_INFO << "Using precompiled kernel.";
205 return fatbin;
206 }
207 }
208
209 string source_path = path_get("source");
210 const string source_md5 = path_files_md5_hash(source_path);
211
212 string common_cflags = compile_kernel_get_common_cflags(kernel_features);
213 const string kernel_md5 = util_md5_string(source_md5 + common_cflags);
214
215 const string include_path = source_path;
216 const string fatbin_file = string_printf(
217 "cycles_%s_%s_%s.hipfb", name, arch.c_str(), kernel_md5.c_str());
218 const string fatbin = path_cache_get(path_join("kernels", fatbin_file));
219 const string hiprt_include_path = path_join(source_path, "kernel/device/hiprt");
220
221 LOG_INFO << "Testing for locally compiled kernel " << fatbin << ".";
222 if (path_exists(fatbin)) {
223 LOG_INFO << "Using locally compiled kernel.";
224 return fatbin;
225 }
226
227# ifdef _WIN32
228 if (!use_adaptive_compilation() && have_precompiled_kernels()) {
229 if (!hipSupportsDevice(hipDevId)) {
230 set_error(
231 string_printf("HIP backend requires compute capability 10.1 or up, but found %d.%d. "
232 "Your GPU is not supported.",
233 major,
234 minor));
235 }
236 else {
237 set_error(
238 string_printf("HIP binary kernel for this graphics card compute "
239 "capability (%d.%d) not found.",
240 major,
241 minor));
242 }
243 return string();
244 }
245# endif
246
247 const char *const hipcc = hipewCompilerPath();
248 if (hipcc == nullptr) {
249 set_error(
250 "HIP hipcc compiler not found. "
251 "Install HIP toolkit in default location.");
252 return string();
253 }
254
255 const int hipcc_hip_version = hipewCompilerVersion();
256 LOG_INFO << "Found hipcc " << hipcc << ", HIP version " << hipcc_hip_version << ".";
257 if (hipcc_hip_version < 40) {
258 LOG_WARNING << "Unsupported HIP version " << hipcc_hip_version / 10 << "."
259 << hipcc_hip_version % 10 << ", you need HIP 4.0 or newer.\n";
260 return string();
261 }
262
264
265 source_path = path_join(path_join(source_path, "kernel"),
266 path_join("device", path_join(base, string_printf("%s.cpp", name))));
267
268 const char *const kernel_ext = "genco";
269 string options;
270 options.append("-Wno-parentheses-equality -Wno-unused-value -ffast-math -O3 -std=c++17");
271 options.append(" --offload-arch=").append(arch.c_str());
272
273 LOG_INFO_IMPORTANT << "Compiling " << source_path << " and caching to " << fatbin;
274
275 double starttime = time_dt();
276
277 string compile_command = string_printf("%s %s -I %s -I %s --%s %s -o \"%s\" %s",
278 hipcc,
279 options.c_str(),
280 include_path.c_str(),
281 hiprt_include_path.c_str(),
282 kernel_ext,
283 source_path.c_str(),
284 fatbin.c_str(),
285 common_cflags.c_str());
286
287 LOG_INFO_IMPORTANT << "Compiling " << ((use_adaptive_compilation()) ? "adaptive " : "")
288 << "HIP-RT kernel ... " << compile_command;
289
290# ifdef _WIN32
291 compile_command = "call " + compile_command;
292# endif
293 if (system(compile_command.c_str()) != 0) {
294 set_error(
295 "Failed to execute linking command, "
296 "see console for details.");
297 return string();
298 }
299
300 LOG_INFO_IMPORTANT << "Kernel compilation finished in " << std::fixed << std::setprecision(2)
301 << time_dt() - starttime << "s";
302
303 return fatbin;
304}
305
306bool HIPRTDevice::load_kernels(const uint64_t kernel_features)
307{
308 if (hiprt_module_) {
309 if (use_adaptive_compilation()) {
310 LOG_INFO << "Skipping HIP kernel reload for adaptive compilation, not currently supported.";
311 }
312 return true;
313 }
314
315 if (hipContext == nullptr) {
316 return false;
317 }
318
319 if (!support_device(kernel_features)) {
320 return false;
321 }
322
323 /* Keep track of whether motion blur is enabled, so to enable/disable motion in BVH builds
324 * This is necessary since objects may be reported to have motion if the Vector pass is
325 * active, but may still need to be rendered without motion blur if that isn't active as well.
326 */
327 use_motion_blur = use_motion_blur || (kernel_features & KERNEL_FEATURE_OBJECT_MOTION);
328
329 /* get kernel */
330 const char *kernel_name = "kernel";
331 string fatbin = compile_kernel(kernel_features, kernel_name);
332 if (fatbin.empty()) {
333 return false;
334 }
335
336 /* open module */
337 HIPContextScope scope(this);
338
339 string fatbin_data;
340 hipError_t result;
341
342 if (path_read_compressed_text(fatbin, fatbin_data)) {
343 result = hipModuleLoadData(&hiprt_module_, fatbin_data.c_str());
344 }
345 else {
346 result = hipErrorFileNotFound;
347 }
348
349 if (result != hipSuccess) {
350 set_error(string_printf(
351 "Failed to load HIP kernel from '%s' (%s)", fatbin.c_str(), hipewErrorString(result)));
352 }
353
354 if (result != hipSuccess) {
355 return false;
356 }
357
358 kernels.load_raytrace(this, hiprt_module_);
359
360 return HIPDevice::load_kernels(kernel_features);
361}
362
363void HIPRTDevice::const_copy_to(const char *name, void *host, const size_t size)
364{
365 /* Set constant memory for HIP module. */
366 HIPDevice::const_copy_to(name, host, size);
367
368 HIPContextScope scope(this);
369 hipDeviceptr_t mem;
370 size_t bytes;
371
372 if (strcmp(name, "data") == 0) {
373 assert(size <= sizeof(KernelData));
374 KernelData *const data = (KernelData *)host;
375 *(hiprtScene *)&data->device_bvh = scene;
376 }
377
378 hip_assert(hipModuleGetGlobal(&mem, &bytes, hiprt_module_, "kernel_params"));
379 assert(bytes == sizeof(KernelParamsHIPRT));
380
381# define KERNEL_DATA_ARRAY(data_type, data_name) \
382 if (strcmp(name, #data_name) == 0) { \
383 hip_assert(hipMemcpyHtoD(mem + offsetof(KernelParamsHIPRT, data_name), host, size)); \
384 return; \
385 }
386 KERNEL_DATA_ARRAY(KernelData, data)
387 KERNEL_DATA_ARRAY(IntegratorStateGPU, integrator_state)
388 KERNEL_DATA_ARRAY(int, user_instance_id)
389 KERNEL_DATA_ARRAY(uint64_t, blas_ptr)
390
391# include "kernel/data_arrays.h"
392# undef KERNEL_DATA_ARRAY
393}
394
395hiprtGeometryBuildInput HIPRTDevice::prepare_triangle_blas(BVHHIPRT *bvh, Mesh *mesh)
396{
397 hiprtGeometryBuildInput geom_input;
398 geom_input.geomType = Triangle;
399
400 if (use_motion_blur && mesh->has_motion_blur()) {
401 const Attribute *attr_P = mesh->attributes.find(ATTR_STD_POSITION);
402 const size_t num_triangles = mesh->num_triangles();
403 int num_bounds = 0;
404
405 bvh->custom_primitive_bound.alloc(num_triangles);
406 for (uint j = 0; j < num_triangles; j++) {
407 Mesh::Triangle t = mesh->get_triangle(j);
409 for (int attr_step = 0; attr_step < attr_P->num_motion_steps(); attr_step++) {
410 t.bounds_grow(attr_P->data<packed_float3>(attr_step), bounds);
411 }
412
413 bvh->custom_primitive_bound[num_bounds] = bounds;
414 num_bounds++;
415 }
416
417 bvh->custom_prim_aabb.aabbCount = num_bounds;
418 bvh->custom_prim_aabb.aabbStride = sizeof(BoundBox);
419 bvh->custom_primitive_bound.copy_to_device();
420 bvh->custom_prim_aabb.aabbs = (void *)bvh->custom_primitive_bound.device_pointer;
421
422 geom_input.type = hiprtPrimitiveTypeAABBList;
423 geom_input.primitive.aabbList = bvh->custom_prim_aabb;
424 geom_input.geomType = Motion_Triangle;
425
426 if (bvh->custom_primitive_bound.device_pointer == 0) {
427 set_error("Failed to allocate triangle custom_primitive_bound for BLAS");
428 }
429 }
430 else {
431 size_t triangle_size = mesh->get_triangles().size();
432 int *triangle_data = mesh->get_triangles().data();
433
434 size_t vertex_size = mesh->num_verts();
435 const packed_float3 *verts = mesh->get_position();
436
437 bvh->triangle_mesh.triangleCount = mesh->num_triangles();
438 bvh->triangle_mesh.triangleStride = 3 * sizeof(int);
439 bvh->triangle_mesh.vertexCount = vertex_size;
440 bvh->triangle_mesh.vertexStride = sizeof(packed_float3);
441
442 /* TODO: reduce memory usage by avoiding copy. */
443 int *triangle_index_data = bvh->triangle_index.resize(triangle_size);
444 float *vertex_data_data = bvh->vertex_data.resize(vertex_size * 3);
445
446 if (triangle_index_data && vertex_data_data) {
447 std::copy_n(triangle_data, triangle_size, triangle_index_data);
448 std::copy_n(verts, vertex_size, reinterpret_cast<packed_float3 *>(vertex_data_data));
449
450 bvh->triangle_index.copy_to_device();
451 bvh->vertex_data.copy_to_device();
452 }
453
454 bvh->triangle_mesh.triangleIndices = (void *)(bvh->triangle_index.device_pointer);
455 bvh->triangle_mesh.vertices = (void *)(bvh->vertex_data.device_pointer);
456
457 geom_input.type = hiprtPrimitiveTypeTriangleMesh;
458 geom_input.primitive.triangleMesh = bvh->triangle_mesh;
459
460 if (bvh->triangle_index.device_pointer == 0 || bvh->vertex_data.device_pointer == 0) {
461 set_error("Failed to allocate triangle data for BLAS");
462 }
463 }
464
465 return geom_input;
466}
467
468hiprtGeometryBuildInput HIPRTDevice::prepare_curve_blas(BVHHIPRT *bvh, Hair *hair)
469{
470 hiprtGeometryBuildInput geom_input;
471
472 const size_t num_curves = hair->num_curves();
473 const size_t num_segments = hair->num_segments();
474 const Attribute *attr_P = hair->attributes.find(ATTR_STD_POSITION);
475 const Attribute *attr_R = hair->attributes.find(ATTR_STD_RADIUS);
476 const bool has_motion = use_motion_blur && hair->has_motion_blur() && attr_P->has_motion();
477
478 bvh->custom_primitive_bound.alloc(num_segments);
479
480 int num_bounds = 0;
481
482 for (uint j = 0; j < num_curves; j++) {
483 const Hair::Curve curve = hair->get_curve(j);
484 const float *curve_radius = hair->get_radius();
485 for (int k = 0; k < curve.num_keys - 1; k++) {
486 if (!has_motion) {
488 curve.bounds_grow(k, hair->get_position(), curve_radius, bounds);
489
490 bvh->custom_primitive_bound[num_bounds] = bounds;
491 num_bounds++;
492 }
493 else {
495 for (int attr_step = 0; attr_step < attr_P->num_motion_steps(); attr_step++) {
496 curve.bounds_grow(
497 k, attr_P->data<packed_float3>(attr_step), attr_R->data<float>(attr_step), bounds);
498 }
499 bvh->custom_primitive_bound[num_bounds] = bounds;
500 num_bounds++;
501 }
502 }
503 }
504
505 bvh->custom_prim_aabb.aabbCount = num_bounds;
506 bvh->custom_prim_aabb.aabbStride = sizeof(BoundBox);
507 bvh->custom_primitive_bound.copy_to_device();
508 bvh->custom_prim_aabb.aabbs = (void *)bvh->custom_primitive_bound.device_pointer;
509
510 geom_input.type = hiprtPrimitiveTypeAABBList;
511 geom_input.primitive.aabbList = bvh->custom_prim_aabb;
512 geom_input.geomType = Curve;
513
514 if (bvh->custom_primitive_bound.device_pointer == 0) {
515 set_error("Failed to allocate curve custom_primitive_bound for BLAS");
516 }
517
518 return geom_input;
519}
520
521hiprtGeometryBuildInput HIPRTDevice::prepare_point_blas(BVHHIPRT *bvh, PointCloud *pointcloud)
522{
523 hiprtGeometryBuildInput geom_input;
524
525 const Attribute *attr_P = nullptr;
526 const Attribute *attr_R = nullptr;
527 if (use_motion_blur && pointcloud->has_motion_blur()) {
528 attr_P = pointcloud->attributes.find(ATTR_STD_POSITION);
529 attr_R = pointcloud->attributes.find(ATTR_STD_RADIUS);
530 if (!attr_P->has_motion()) {
531 attr_P = nullptr;
532 attr_R = nullptr;
533 }
534 }
535
536 const packed_float3 *points_data = pointcloud->get_position();
537 const float *radius_data = pointcloud->get_radius();
538 const size_t num_points = pointcloud->num_points();
539
540 int num_bounds = 0;
541
542 if (attr_P == nullptr) {
543 bvh->custom_primitive_bound.alloc(num_points);
544 for (uint j = 0; j < num_points; j++) {
545 const PointCloud::Point point = pointcloud->get_point(j);
547 point.bounds_grow(points_data, radius_data, bounds);
548
549 bvh->custom_primitive_bound[num_bounds] = bounds;
550 num_bounds++;
551 }
552 }
553 else {
554 bvh->custom_primitive_bound.alloc(num_points);
555
556 for (uint j = 0; j < num_points; j++) {
557 const PointCloud::Point point = pointcloud->get_point(j);
559 for (int attr_step = 0; attr_step < attr_P->num_motion_steps(); attr_step++) {
560 point.bounds_grow(
561 attr_P->data<packed_float3>(attr_step), attr_R->data<float>(attr_step), bounds);
562 }
563
564 bvh->custom_primitive_bound[num_bounds] = bounds;
565 num_bounds++;
566 }
567 }
568
569 bvh->custom_prim_aabb.aabbCount = num_bounds;
570 bvh->custom_prim_aabb.aabbStride = sizeof(BoundBox);
571 bvh->custom_primitive_bound.copy_to_device();
572 bvh->custom_prim_aabb.aabbs = (void *)bvh->custom_primitive_bound.device_pointer;
573
574 geom_input.type = hiprtPrimitiveTypeAABBList;
575 geom_input.primitive.aabbList = bvh->custom_prim_aabb;
576 geom_input.geomType = Point;
577
578 if (bvh->custom_primitive_bound.device_pointer == 0) {
579 set_error("Failed to allocate point custom_primitive_bound for BLAS");
580 }
581
582 return geom_input;
583}
584
585void HIPRTDevice::build_blas(BVHHIPRT *bvh, Geometry *geom)
586{
587 hiprtGeometryBuildInput geom_input = {};
588
589 hiprtBuildOptions options = {
590 .buildFlags = hiprtBuildFlagBitPreferHighQualityBuild,
591 };
592
593 switch (geom->geometry_type) {
594 case Geometry::MESH:
595 case Geometry::VOLUME: {
596 Mesh *mesh = static_cast<Mesh *>(geom);
597 if (mesh->num_triangles() == 0) {
598 return;
599 }
600 geom_input = prepare_triangle_blas(bvh, mesh);
601 break;
602 }
603
604 case Geometry::HAIR: {
605 Hair *const hair = static_cast<Hair *const>(geom);
606 if (hair->num_segments() == 0) {
607 return;
608 }
609 geom_input = prepare_curve_blas(bvh, hair);
610 options.buildFlags = hiprtBuildFlagBitPreferBalancedBuild;
611 break;
612 }
613
615 PointCloud *pointcloud = static_cast<PointCloud *>(geom);
616 if (pointcloud->num_points() == 0) {
617 return;
618 }
619 geom_input = prepare_point_blas(bvh, pointcloud);
620 options.buildFlags = hiprtBuildFlagBitPreferBalancedBuild;
621 break;
622 }
623
629 return;
630
631 default:
632 assert(geom_input.geomType != hiprtInvalidValue);
633 }
634
635 if (have_error()) {
636 return;
637 }
638
639 size_t blas_scratch_buffer_size = 0;
640 hiprtError rt_err = hiprtGetGeometryBuildTemporaryBufferSize(
641 hiprt_context, geom_input, options, blas_scratch_buffer_size);
642
643 if (rt_err != hiprtSuccess) {
644 set_error("Failed to get scratch buffer size for BLAS");
645 return;
646 }
647
648 rt_err = hiprtCreateGeometry(hiprt_context, geom_input, options, bvh->hiprt_geom);
649
650 if (rt_err != hiprtSuccess) {
651 set_error("Failed to create BLAS");
652 return;
653 }
654 {
655 thread_scoped_lock lock(hiprt_mutex);
656 if (blas_scratch_buffer_size > scratch_buffer_size) {
657 scratch_buffer.alloc(blas_scratch_buffer_size);
658 scratch_buffer.zero_to_device();
659 if (!scratch_buffer.device_pointer) {
660 hiprtDestroyGeometry(hiprt_context, bvh->hiprt_geom);
661 bvh->hiprt_geom = nullptr;
662 set_error("Failed to allocate scratch buffer for BLAS");
663 return;
664 }
665 scratch_buffer_size = blas_scratch_buffer_size;
666 }
667 bvh->geom_input = geom_input;
668 rt_err = hiprtBuildGeometry(hiprt_context,
669 hiprtBuildOperationBuild,
670 bvh->geom_input,
671 options,
672 (void *)(scratch_buffer.device_pointer),
673 nullptr,
674 bvh->hiprt_geom);
675 }
676 if (rt_err != hiprtSuccess) {
677 set_error("Failed to build BLAS");
678 }
679}
680
681hiprtScene HIPRTDevice::build_tlas(BVHHIPRT * /*bvh*/, const vector<Object *> &objects, bool refit)
682{
683
684 size_t num_object = objects.size();
685 if (num_object == 0) {
686 return nullptr;
687 }
688
689 const hiprtBuildOptions options = {
690 .buildFlags = hiprtBuildFlagBitPreferHighQualityBuild,
691 };
692
693 hiprtBuildOperation build_operation = refit ? hiprtBuildOperationUpdate :
694 hiprtBuildOperationBuild;
695
696 array<hiprtFrameMatrix> transform_matrix;
697
698 size_t num_instances = 0;
699 int blender_instance_id = 0;
700
701 user_instance_id.alloc(num_object);
702 prim_visibility.alloc(num_object);
703 hiprt_blas_ptr.alloc(num_object);
704 blas_ptr.alloc(num_object);
705 transform_headers.alloc(num_object);
706
707 for (Object *ob : objects) {
708 uint32_t mask = 0;
709 if (ob->is_traceable()) {
710 mask = ob->visibility_for_tracing();
711 }
712
713 Transform current_transform = ob->get_tfm();
714 Geometry *geom = ob->get_geometry();
715 bool transform_applied = geom->transform_applied;
716
717 BVHHIPRT *current_bvh = static_cast<BVHHIPRT *>(geom->bvh.get());
718 bool is_valid_geometry = current_bvh->geom_input.geomType != hiprtInvalidValue;
719 hiprtGeometry hiprt_geom_current = current_bvh->hiprt_geom;
720
721 hiprtFrameMatrix hiprt_transform_matrix = {{{0}}};
722 Transform identity_matrix = transform_identity();
723 get_hiprt_transform(hiprt_transform_matrix.matrix, identity_matrix);
724
725 if (is_valid_geometry) {
726 hiprtTransformHeader current_header = {0};
727 current_header.frameCount = 1;
728 current_header.frameIndex = transform_matrix.size();
729 if (use_motion_blur && ob->get_motion().size()) {
730 int motion_size = ob->get_motion().size();
731 assert(motion_size != 1);
732
733 array<Transform> tfm_array = ob->get_motion();
734 float time_interval = 1 / (float)(motion_size - 1);
735 current_header.frameCount = motion_size;
736
737 vector<hiprtFrameMatrix> tfm_hiprt_mb;
738 tfm_hiprt_mb.resize(motion_size);
739 for (int i = 0; i < motion_size; i++) {
740 get_hiprt_transform(tfm_hiprt_mb[i].matrix, tfm_array[i]);
741 tfm_hiprt_mb[i].time = (float)i * time_interval;
742 transform_matrix.push_back_slow(tfm_hiprt_mb[i]);
743 }
744 }
745 else {
746 if (transform_applied) {
747 current_transform = identity_matrix;
748 }
749 get_hiprt_transform(hiprt_transform_matrix.matrix, current_transform);
750 transform_matrix.push_back_slow(hiprt_transform_matrix);
751 }
752
753 transform_headers[num_instances] = current_header;
754
755 user_instance_id[num_instances] = blender_instance_id;
756 prim_visibility[num_instances] = mask;
757 hiprt_blas_ptr[num_instances].geometry = hiprt_geom_current;
758 hiprt_blas_ptr[num_instances].type = hiprtInstanceTypeGeometry;
759 num_instances++;
760 }
761 blas_ptr[blender_instance_id] = (uint64_t)hiprt_geom_current;
762 blender_instance_id++;
763 }
764
765 size_t table_ptr_size = 0;
766 hipDeviceptr_t table_device_ptr;
767
768 hip_assert(
769 hipModuleGetGlobal(&table_device_ptr, &table_ptr_size, hiprt_module_, "kernel_params"));
770 if (have_error()) {
771 return nullptr;
772 }
773
774 size_t kernel_param_offset[4];
775 int table_index = 0;
776 kernel_param_offset[table_index++] = offsetof(KernelParamsHIPRT, table_closest_intersect);
777 kernel_param_offset[table_index++] = offsetof(KernelParamsHIPRT, table_shadow_intersect);
778 kernel_param_offset[table_index++] = offsetof(KernelParamsHIPRT, table_local_intersect);
779 kernel_param_offset[table_index++] = offsetof(KernelParamsHIPRT, table_volume_intersect);
780
781 for (int index = 0; index < table_index; index++) {
782 hip_assert(hipMemcpyHtoD(table_device_ptr + kernel_param_offset[index],
783 (void *)&functions_table,
784 sizeof(device_ptr)));
785 if (have_error()) {
786 return nullptr;
787 }
788 }
789
790 if (num_instances == 0) {
791 return nullptr;
792 }
793
794 int frame_count = transform_matrix.size();
795 hiprtSceneBuildInput scene_input_ptr = {nullptr};
796 scene_input_ptr.instanceCount = num_instances;
797 scene_input_ptr.frameCount = frame_count;
798 scene_input_ptr.frameType = hiprtFrameTypeMatrix;
799
800 user_instance_id.copy_to_device();
801 prim_visibility.copy_to_device();
802 hiprt_blas_ptr.copy_to_device();
803 blas_ptr.copy_to_device();
804 transform_headers.copy_to_device();
805
806 if (user_instance_id.device_pointer == 0 || prim_visibility.device_pointer == 0 ||
807 hiprt_blas_ptr.device_pointer == 0 || blas_ptr.device_pointer == 0 ||
808 transform_headers.device_pointer == 0)
809 {
810 set_error("Failed to allocate object buffers for TLAS");
811 return nullptr;
812 }
813
814 {
815 /* TODO: reduce memory usage by avoiding copy. */
816 hiprtFrameMatrix *instance_transform_matrix_data = instance_transform_matrix.resize(
817 frame_count);
818 if (instance_transform_matrix_data == nullptr) {
819 set_error("Failed to allocate host instance_transform_matrix for TLAS");
820 return nullptr;
821 }
822
823 std::copy_n(transform_matrix.data(), frame_count, instance_transform_matrix_data);
824 instance_transform_matrix.copy_to_device();
825
826 if (instance_transform_matrix.device_pointer == 0) {
827 set_error("Failed to allocate instance_transform_matrix for TLAS");
828 return nullptr;
829 }
830 }
831
832 scene_input_ptr.instanceMasks = (void *)prim_visibility.device_pointer;
833 scene_input_ptr.instances = (void *)hiprt_blas_ptr.device_pointer;
834 scene_input_ptr.instanceTransformHeaders = (void *)transform_headers.device_pointer;
835 scene_input_ptr.instanceFrames = (void *)instance_transform_matrix.device_pointer;
836
837 hiprtScene scene = nullptr;
838
839 hiprtError rt_err = hiprtCreateScene(hiprt_context, scene_input_ptr, options, scene);
840
841 if (rt_err != hiprtSuccess) {
842 set_error("Failed to create TLAS");
843 return nullptr;
844 }
845
846 size_t tlas_scratch_buffer_size;
847 rt_err = hiprtGetSceneBuildTemporaryBufferSize(
848 hiprt_context, scene_input_ptr, options, tlas_scratch_buffer_size);
849
850 if (rt_err != hiprtSuccess) {
851 set_error("Failed to get scratch buffer size for TLAS");
852 hiprtDestroyScene(hiprt_context, scene);
853 return nullptr;
854 }
855
856 if (tlas_scratch_buffer_size > scratch_buffer_size) {
857 scratch_buffer.alloc(tlas_scratch_buffer_size);
858 scratch_buffer.zero_to_device();
859 if (scratch_buffer.device_pointer == 0) {
860 set_error("Failed to allocate scratch buffer for TLAS");
861 hiprtDestroyScene(hiprt_context, scene);
862 return nullptr;
863 }
864 }
865
866 rt_err = hiprtBuildScene(hiprt_context,
867 build_operation,
868 scene_input_ptr,
869 options,
870 (void *)scratch_buffer.device_pointer,
871 nullptr,
872 scene);
873
874 scratch_buffer.free();
875 scratch_buffer_size = 0;
876
877 if (rt_err != hiprtSuccess) {
878 set_error("Failed to build TLAS");
879 hiprtDestroyScene(hiprt_context, scene);
880 return nullptr;
881 }
882
883 return scene;
884}
885
886void HIPRTDevice::free_bvh_memory_delayed()
887{
888 thread_scoped_lock lock(hiprt_mutex);
889 if (stale_bvh.size()) {
890 for (int bvh_index = 0; bvh_index < stale_bvh.size(); bvh_index++) {
891 hiprtGeometry hiprt_geom = stale_bvh[bvh_index];
892 hiprtDestroyGeometry(hiprt_context, hiprt_geom);
893 hiprt_geom = nullptr;
894 }
895 stale_bvh.clear();
896 }
897}
898
899void HIPRTDevice::release_bvh(BVH *bvh)
900{
901 BVHHIPRT *current_bvh = static_cast<BVHHIPRT *>(bvh);
902 thread_scoped_lock lock(hiprt_mutex);
903 /* Tracks BLAS pointers whose BVH destructors have been called. */
904 stale_bvh.push_back(current_bvh->hiprt_geom);
905}
906
907void HIPRTDevice::build_bvh(BVH *bvh, Progress &progress, bool refit)
908{
909 if (have_error()) {
910 return;
911 }
912 free_bvh_memory_delayed();
913 progress.set_substatus("Building HIPRT acceleration structure");
914
915 BVHHIPRT *bvh_rt = static_cast<BVHHIPRT *>(bvh);
916 HIPContextScope scope(this);
917
918 if (!bvh_rt->is_tlas()) {
919 const vector<Geometry *> &geometry = bvh_rt->geometry;
920 assert(geometry.size() == 1);
921 build_blas(bvh_rt, geometry[0]);
922 }
923 else {
924 if (scene) {
925 hiprtDestroyScene(hiprt_context, scene);
926 scene = nullptr;
927 }
928 scene = build_tlas(bvh_rt, bvh_rt->objects, refit);
929 }
930}
932
933#endif
volatile int lock
unsigned long long int uint64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition btDbvt.cpp:52
static btDbvtVolume bounds(btDbvtNode **leaves, int count)
Definition btDbvt.cpp:299
void refit(btStridingMeshInterface *triangles, const btVector3 &aabbMin, const btVector3 &aabbMax)
Attribute * find(ustring name) const
const T * data(const int step=0) const
bool has_motion() const
int num_motion_steps() const
Definition bvh/bvh.h:67
Type geometry_type
bool transform_applied
const packed_float3 * get_position() const
virtual bool has_motion_blur() const
unique_ptr< BVH > bvh
const float * get_radius() const
AttributeSet attributes
Definition hair.h:13
Curve get_curve(const size_t i) const
Definition hair.h:94
size_t num_curves() const
Definition hair.h:109
size_t num_segments() const
Definition hair.h:114
NODE_DECLARE size_t num_verts() const
bool has_motion_blur() const override
size_t num_triangles() const
Definition scene/mesh.h:73
Triangle get_triangle(const size_t i) const
Definition scene/mesh.h:67
Point get_point(const int i) const
size_t num_points() const
void set_substatus(const string &substatus_)
Definition progress.h:259
size_t size() const
void push_back_slow(const T &t)
@ MEM_READ_WRITE
@ MEM_DEVICE_ONLY
@ MEM_READ_ONLY
CCL_NAMESPACE_BEGIN struct Options options
#define KERNEL_DATA_ARRAY(type, name)
Definition data_arrays.h:8
#define KERNEL_FEATURE_OBJECT_MOTION
#define CCL_NAMESPACE_END
#define offsetof(t, d)
#define assert(assertion)
#define index(i)
@ BVH_LAYOUT_HIPRT
@ ATTR_STD_POSITION
@ ATTR_STD_RADIUS
#define LOG_DEBUG
Definition log.h:107
#define LOG_INFO_IMPORTANT
Definition log.h:105
#define LOG_IS_ON(level)
Definition log.h:113
@ LOG_LEVEL_TRACE
Definition log.h:27
#define LOG_WARNING
Definition log.h:103
#define LOG_INFO
Definition log.h:106
ccl_device_inline float2 mask(const MaskType mask, const float2 a)
string util_md5_string(const string &str)
Definition md5.cpp:386
unsigned int uint32_t
Definition BKE_text.h:81
BMesh const char void * data
int BVHLayoutMask
Definition params.h:50
string path_cache_get(const string &sub)
Definition path.cpp:383
bool path_create_directories(const string &filepath)
Definition path.cpp:696
string path_get(const string &sub)
Definition path.cpp:368
string path_files_md5_hash(const string &dir)
Definition path.cpp:660
string path_join(const string &dir, const string &file)
Definition path.cpp:436
bool path_exists(const string &path)
Definition path.cpp:603
bool path_read_compressed_text(const string &path, string &text)
Definition path.cpp:803
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
CCL_NAMESPACE_BEGIN string string_printf(const char *format,...)
Definition string.cpp:25
void bounds_grow(const int k, const packed_float3 *curve_keys, const float *curve_radius, BoundBox &bounds) const
Definition hair.cpp:44
int num_keys
Definition hair.h:20
void bounds_grow(const packed_float3 *verts, BoundBox &bounds) const
void bounds_grow(const packed_float3 *points, const float *radius, BoundBox &bounds) const
float4 y
Definition transform.h:53
float4 x
Definition transform.h:53
float4 z
Definition transform.h:53
float y
Definition sky_math.h:225
float z
Definition sky_math.h:225
float x
Definition sky_math.h:225
float w
Definition sky_math.h:225
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
CCL_NAMESPACE_BEGIN double time_dt()
Definition time.cpp:31
ccl_device_inline Transform transform_identity()
Definition transform.h:355
uint64_t device_ptr
Definition types_base.h:44