12#include <shared_mutex>
33 using Island = std::span<SubDevice *const>;
46 auto it = sub.
ptr_map.find(key);
53 return sub.
ptr_map.contains(key);
94 if (sub.peer_island_index < 0) {
106 if (peer_sub.peer_island_index < 0 &&
107 peer_sub.device->info.type == sub.device->info.type &&
108 peer_sub.device->check_peer_access(sub.device.get()))
110 peer_sub.peer_island_index = sub.peer_island_index;
111 peer_islands[sub.peer_island_index].push_back(&peer_sub);
122 bool have_disabled_hardware_rt =
false;
123 bool have_enabled_hardware_rt =
false;
128 have_enabled_hardware_rt =
true;
131 have_disabled_hardware_rt =
true;
150 error_msg += sub.device->error_message();
161 BVHLayoutMask device_bvh_layout_mask = sub_device.device->get_bvh_layout_mask(
163 bvh_layout_mask &= device_bvh_layout_mask;
164 bvh_layout_mask_all |= device_bvh_layout_mask;
188 if ((bvh_layout_mask_all & BVH_LAYOUT_OPTIX_EMBREE) == BVH_LAYOUT_OPTIX_EMBREE) {
192 if ((bvh_layout_mask_all & BVH_LAYOUT_METAL_EMBREE) == BVH_LAYOUT_METAL_EMBREE) {
196 if ((bvh_layout_mask_all & BVH_LAYOUT_EMBREEGPU_EMBREE) == BVH_LAYOUT_EMBREEGPU_EMBREE) {
201 if ((bvh_layout_mask_all & BVH_LAYOUT_HIPRT_EMBREE) == BVH_LAYOUT_HIPRT_EMBREE) {
205 return bvh_layout_mask;
211 if (!sub.device->load_kernels(kernel_features)) {
222 if (!sub.device->load_osl_kernels()) {
253 geom_bvhs.reserve(bvh->
geometry.size());
255 geom_bvhs.push_back(std::move(geom->
bvh));
262 for (
size_t k = 0; k < bvh->
geometry.size(); ++k) {
265 static_cast<BVHMulti *
>(geom_bvhs[k].get())->sub_bvhs[i].get());
311 sub.device->build_bvh(bvh_multi->
sub_bvhs[i].get(), progress,
refit);
316 for (
size_t k = 0; k < bvh->
geometry.size(); ++k) {
318 bvh->
geometry[k]->bvh = std::move(geom_bvhs[k]);
329 return devices.back().device->get_cpu_osl_memory();
334 if (mem.
device == sub_device) {
344 if (sub.device.get() == sub_device) {
351 assert(!
"MultiDevice::mem_device_ptr could not find sub_device");
366 if (sub.device.get() == sub_device) {
382 owner_sub = island_sub;
397 (island_sub->device->stats.mem_used < owner_sub->
device->stats.mem_used))
399 owner_sub = island_sub;
416 islands.emplace_back(peer_island);
420 for (
SubDevice *
const &sub : peer_island) {
421 islands.emplace_back(&sub, 1);
431 return sub.device->host_alloc(type, size);
442 sub.device->host_free(type, host_pointer, size);
462 owner_sub->
device->mem_alloc(mem);
483 owner_sub->
device->mem_copy_to(mem);
489 if (island_sub != owner_sub) {
490 island_sub->device->mem_copy_to(mem);
515 if (!owner_sub->
device->is_shared(
518 owner_sub->
device->mem_move_to_host(mem);
523 if (island_sub != owner_sub) {
524 island_sub->device->mem_move_to_host(mem);
542 if (sub.device.get() == sub_device) {
549 return owner_sub->
device->is_shared(shared_pointer, ptr, owner_sub->
device.get());
553 assert(!
"is_shared failed to find matching device");
565 owner_sub->
device->mem_or_from_device(mem);
573 device_memory &mem,
const size_t y,
size_t w,
const size_t h,
size_t elem)
override
576 const size_t sub_h = h /
devices.size();
580 size_t sy =
y + i * sub_h;
581 size_t sh = (i == (size_t)
devices.size() - 1) ? h - sub_h * i : sub_h;
587 owner_sub->
device->mem_copy_from(mem, sy,
w, sh, elem);
607 owner_sub->
device->mem_zero(mem);
628 owner_sub->
device->mem_free(mem);
634 if (island_sub != owner_sub) {
635 island_sub->device->mem_free(mem);
647 void const_copy_to(
const char *name,
void *host,
const size_t size)
override
650 sub.device->const_copy_to(name, host, size);
659 if (sub.device.get() == sub_device) {
671 sub.device->foreach_device(callback);
678 if (!sub.device->has_unified_image_memory_all()) {
691 return make_unique<MultiDevice>(info, stats, profiler, headless);
unsigned long long int uint64_t
void refit(btStridingMeshInterface *triangles, const btVector3 &aabbMin, const btVector3 &aabbMax)
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
vector< unique_ptr< BVH > > sub_bvhs
static unique_ptr< BVH > create(const BVHParams ¶ms, const vector< Geometry * > &geometry, const vector< Object * > &objects, Device *device)
vector< Geometry * > geometry
vector< Object * > objects
vector< DeviceInfo > multi_devices
bool use_hardware_raytracing
virtual void host_free(const MemoryType type, void *host_pointer, const size_t size)
Device(const DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool headless_)
void image_load_requested_gpu(DeviceQueue &queue)
friend class device_memory
static unique_ptr< Device > create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
virtual void * host_alloc(const MemoryType type, const size_t size)
void mem_or_from_device(device_memory &mem) override
bool has_unified_image_memory_all() const override
list< SubDevice > devices
std::atomic< device_ptr > unique_key
void ptr_map_erase(SubDevice &sub, const device_ptr key)
device_ptr ptr_map_find(const SubDevice &sub, const device_ptr key) const
BVHLayoutMask get_bvh_layout_mask(const uint64_t kernel_features) const override
bool is_resident(device_ptr key, Device *sub_device) override
bool load_osl_kernels() override
std::shared_mutex ptr_map_mutex
void ptr_map_set(SubDevice &sub, const device_ptr key, const device_ptr ptr)
const string & error_message() override
int device_number(Device *sub_device) override
void verify_hardware_raytracing()
void mem_copy_to(device_memory &mem) override
bool load_kernels(const uint64_t kernel_features) override
void mem_free(device_memory &mem) override
void * host_alloc(const MemoryType type, const size_t size) override
void set_image_cache_func(KernelImageLoadRequestedCPU image_load_requested_cpu, KernelImageLoadRequestedGPU image_load_requested_gpu) override
vector< vector< SubDevice * > > peer_islands
void mem_zero(device_memory &mem) override
bool is_shared(const void *shared_pointer, const device_ptr key, Device *sub_device) override
void build_bvh(BVH *bvh, Progress &progress, bool refit) override
void const_copy_to(const char *name, void *host, const size_t size) override
void foreach_device(const std::function< void(Device *)> &callback) override
bool ptr_map_contains(const SubDevice &sub, const device_ptr key) const
std::span< SubDevice *const > Island
SubDevice * find_suitable_mem_device(device_ptr key, const Island island)
void host_free(const MemoryType type, void *host_pointer, const size_t size) override
vector< Island > mem_islands(const device_memory &mem) const
MultiDevice(const DeviceInfo &info_, Stats &stats, Profiler &profiler, bool headless)
void mem_alloc(device_memory &mem) override
device_ptr mem_device_ptr(const device_memory &mem, Device *sub_device) override
void mem_copy_from(device_memory &mem, const size_t y, size_t w, const size_t h, size_t elem) override
OSLGlobals * get_cpu_osl_memory() override
SubDevice * find_matching_mem_device(device_ptr key, SubDevice &sub)
void mem_move_to_host(device_memory &mem) override
device_ptr find_matching_mem(device_ptr key, SubDevice &sub)
void mem_alloc(const size_t size)
void mem_free(const size_t size)
device_ptr device_pointer
#define CCL_NAMESPACE_END
unique_ptr< Device > device_multi_create(const DeviceInfo &info, Stats &stats, Profiler &profiler, bool headless)
#define assert(assertion)
@ BVH_LAYOUT_MULTI_HIPRT_EMBREE
@ BVH_LAYOUT_MULTI_EMBREEGPU
@ BVH_LAYOUT_MULTI_METAL_EMBREE
@ BVH_LAYOUT_MULTI_EMBREEGPU_EMBREE
@ BVH_LAYOUT_MULTI_OPTIX_EMBREE
unique_ptr< Device > device
map< device_ptr, device_ptr > ptr_map
std::function< void(size_t, int, int, int, KernelTileDescriptor &tile_descriptor)> KernelImageLoadRequestedCPU
std::function< void(DeviceQueue &)> KernelImageLoadRequestedGPU