25constexpr auto tie(
const VkImageCreateInfo &info)
27 return std::tie(info.format,
44 const auto &[_1, _2, _3, _4, _5, _6, _7, _8, _9] =
detail::tie(value);
68 const std::string &name_str)
74 VkResult create_result = device.
functions.vkCreateImage(
80 VkResult bind_result = vmaBindImageMemory2(
86 bool use_subresource_tracking = info.
create_info.arrayLayers > 1 ||
91 image, use_subresource_tracking, VK_NULL_HANDLE, name_str.c_str());
93 return {.resource_handle = resource_handle, .vk_handle =
image};
102 VkMemoryRequirements2 reqs_out = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2};
105 VkDeviceImageMemoryRequirements reqs_info = {
106 .sType = VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS,
108 .pCreateInfo = &image_info,
109 .planeAspect = VK_IMAGE_ASPECT_NONE,
111 device.
functions.vkGetDeviceImageMemoryRequirementsKHR(
112 device.
vk_handle(), &reqs_info, &reqs_out);
115 VkImage
image = VK_NULL_HANDLE;
121 VkImageMemoryRequirementsInfo2 reqs_info = {
122 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
126 device.
functions.vkGetImageMemoryRequirements2(device.
vk_handle(), &reqs_info, &reqs_out);
130 return reqs_out.memoryRequirements;
136 VKImageHandle *image_handle = cache_.lookup_ptr(info);
138 image_handle->unused_cycles_count = 0;
139 return image_handle->resource;
143 std::string name_str;
145 name_str = fmt::format(
"VkImageFromPool_{}", cache_.size());
150 cache_.add_new(info, {.resource = resource});
157 VKImageHandle &image_handle = cache_.lookup(info);
158 return std::move(image_handle.image_views);
163 VKImageHandle &image_handle = cache_.lookup(info);
164 image_handle.image_views = std::move(image_views);
169 VKImageHandle *image_handle = cache_.lookup_ptr(info);
171 "Uninitialized VkImage passed to VKImageCache::reset_unused_cycles_count()");
172 image_handle->unused_cycles_count = 0;
179 for (
decltype(cache_)::MutableItem item : cache_.items()) {
180 if (force_reset ==
true || item.value.unused_cycles_count >= max_unused_cycles_) {
181 unused_keys.append(item.key);
184 item.value.unused_cycles_count++;
190 VKImageHandle handle = cache_.pop(key);
201 if (key.allocation == allocation) {
202 unused_keys.append(key);
208 VKImageHandle handle = cache_.pop(key);
214std::optional<VKMemorySegment> VKTexturePool::AllocationHandle::acquire(
215 VkMemoryRequirements requirements)
219 uint32_t memory_type_bit = 1u << allocation_info.memoryType;
220 if (allocation_info.memoryType != 0 && !
bool(requirements.memoryTypeBits & memory_type_bit)) {
226 auto found_segment = std::ranges::find_if(segments, [&](
const VKMemorySegment &segment) {
228 VkDeviceSize remaining_size = segment.size - (aligned_offset - segment.offset);
231 aligned_offset < segment.offset + segment.size &&
233 remaining_size >= requirements.size;
235 if (found_segment == segments.end()) {
241 VkDeviceSize aligned_offset =
ceil_to_multiple_ul(found_segment->offset, requirements.alignment);
242 VKMemorySegment
segment = {.offset = aligned_offset, .size = requirements.size};
243 VKMemorySegment segment_prev = {.offset = found_segment->offset,
244 .size =
segment.offset - found_segment->offset};
245 VKMemorySegment segment_next = {.offset =
segment.offset +
segment.size,
246 .size = found_segment->size -
segment.size - segment_prev.size};
249 if (segment_prev.size > 0 && segment_next.size > 0) {
250 *found_segment = segment_next;
251 segments.insert(found_segment, segment_prev);
253 else if (segment_prev.size > 0) {
254 *found_segment = segment_prev;
256 else if (segment_next.size > 0) {
257 *found_segment = segment_next;
260 segments.erase(found_segment);
265 segment.offset -= allocation_info.offset;
270void VKTexturePool::AllocationHandle::release(VKMemorySegment segment)
273 segment.offset += allocation_info.offset;
276 auto segment_next = std::ranges::find_if(
277 segments, [segment](VKMemorySegment next) {
return segment.offset < next.offset; });
278 auto segment_prev = segment_next;
279 if (segment_prev != segments.begin()) {
284 bool extend_prev = segment_prev != segments.end() &&
285 segment.offset == (segment_prev->offset + segment_prev->size);
286 bool extend_next = segment_next != segments.end() &&
289 segment_prev->size +=
segment.size;
292 segment_next->offset =
segment.offset;
293 segment_next->size +=
segment.size;
298 if (extend_prev && extend_next) {
299 segment_prev->size += segment_next->size -
segment.size;
300 segments.erase(segment_next);
302 else if (!(extend_prev || extend_next)) {
303 segments.insert(segment_next, segment);
307void VKTexturePool::AllocationHandle::alloc(VkMemoryRequirements requirements)
311 VmaAllocationCreateInfo create_info = {
312 .flags = VMA_ALLOCATION_CREATE_CAN_ALIAS_BIT,
313 .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
314 .memoryTypeBits = requirements.memoryTypeBits,
319 create_info.flags |= VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT;
320 create_info.pUserData = (
void *)__func__;
323 VkResult
result = vmaAllocateMemory(
324 device.mem_allocator_get(), &requirements, &create_info, &allocation, &allocation_info);
329 segments = {{.offset = allocation_info.offset, .size = allocation_info.size}};
332void VKTexturePool::AllocationHandle::free()
335 discard_pool.discard_allocation(allocation);
351 image_cache_->reset(
true);
354 for (TextureHandle handle : acquired_) {
355 delete handle.texture;
357 for (AllocationHandle handle : allocations_) {
370 int mip_len_max = 1 +
floorf(log2f(std::max({extent.x, extent.y, extent.z})));
371 mip_len =
min_ii(mip_len, mip_len_max);
381 texture->gpu_image_usage_flags_ = usage;
384 texture->mip_max_ = mip_len - 1;
386 texture->device_format_ = format;
387 if (
texture->device_format_ == TextureFormat::SFLOAT_16_16_16) {
388 texture->device_format_ = TextureFormat::SFLOAT_16_16_16_16;
390 if (
texture->device_format_ == TextureFormat::SFLOAT_32_32_32) {
391 texture->device_format_ = TextureFormat::SFLOAT_32_32_32_32;
399 VkImageCreateInfo create_info = {
400 .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
405 .extent =
texture->vk_extent_3d(0),
408 .samples = VK_SAMPLE_COUNT_1_BIT,
409 .tiling = VK_IMAGE_TILING_OPTIMAL,
411 .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
416 VKImageInfo image_info = {.create_info = create_info};
419 for (AllocationHandle allocation_handle : allocations_) {
420 std::optional<VKMemorySegment> segment_opt = allocation_handle.acquire(requirements);
422 image_info.
allocation = allocation_handle.allocation;
423 image_info.
segment = segment_opt.value();
424 allocation_handle.unused_cycles_count = 0;
425 allocations_.add_overwrite(allocation_handle);
431 if (image_info.
allocation == VK_NULL_HANDLE) {
432 VkMemoryRequirements allocation_requirements = requirements;
433 allocation_requirements.size = std::max(allocation_size, requirements.size);
435 AllocationHandle handle;
436 handle.alloc(allocation_requirements);
438 std::optional<VKMemorySegment> segment_opt = handle.acquire(requirements);
441 image_info.
segment = segment_opt.value();
442 allocations_.add(handle);
450 std::string name_str;
452 name_str = name ? name : fmt::format(
"TexFromPool_{}", acquired_.size());
457 texture->resource_ = image_cache_->get_or_create(image_info);
459 texture->image_views_ = image_cache_->take_image_views(image_info);
464 texture->vk_image_usage_ = create_info.usage;
469 current_usage_data_.acquired_segment_size += image_info.
segment.
size;
470 current_usage_data_.acquired_segment_size_max = std::max(
471 current_usage_data_.acquired_segment_size_max, current_usage_data_.acquired_segment_size);
475 acquired_.add(TextureHandle{.texture =
texture, .image_info = image_info});
483 "Unacquired texture passed to VKTexturePool::offset_users_count()");
485 TextureHandle texture_handle = acquired_.lookup_key({
unwrap(
texture)});
486 VKImageInfo image_info = texture_handle.image_info;
487 AllocationHandle allocation_handle = allocations_.lookup_key({image_info.
allocation});
494 if (context !=
nullptr) {
497 .vk_image_layout = VK_IMAGE_LAYOUT_GENERAL,
500 .vk_access_flags = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT,
501 .reset_layout_to_undefined =
true,
503 context->render_graph().add_node(synchronization);
507 current_usage_data_.acquired_segment_size -= image_info.
segment.
size;
511 allocation_handle.release(image_info.
segment);
512 allocations_.add_overwrite(allocation_handle);
516 image_cache_->store_image_views(image_info, std::move(texture_handle.texture->image_views_));
525 acquired_.remove(texture_handle);
526 delete texture_handle.texture;
532 "Unacquired texture passed to VKTexturePool::offset_users_count()");
533 TextureHandle texture_handle = acquired_.lookup_key({
unwrap(
texture)});
534 texture_handle.users_count += offset;
535 acquired_.add_overwrite(texture_handle);
541 for (
const TextureHandle &tex : acquired_) {
544 image_cache_->reset_unused_cycles_count(tex.image_info);
550 "Missing texture release/retain. Likely TextureFromPool::release(), "
551 "TextureFromPool::retain() or TexturePool::release_texture().");
556 for (AllocationHandle handle : allocations_) {
557 if (handle.is_unused() && (handle.unused_cycles_count >= max_unused_cycles_ || force_free)) {
558 unused_allocations.append(handle);
561 handle.unused_cycles_count++;
562 allocations_.add_overwrite(handle);
567 for (AllocationHandle handle : unused_allocations) {
569 image_cache_->discard_all_of(handle.allocation);
572 allocations_.remove(handle);
577 image_cache_->reset();
583 current_usage_data_.allocation_count = allocations_.size();
584 current_usage_data_.image_cache_size = image_cache_ ? image_cache_->size() : 0;
586 if (!(previous_usage_data_ == current_usage_data_)) {
591 previous_usage_data_ = current_usage_data_;
592 current_usage_data_ = {};
593 for (
const TextureHandle &tex : acquired_) {
594 current_usage_data_.acquired_segment_size += tex.image_info.segment.size;
599void VKTexturePool::log_usage_data()
601 VkDeviceSize total_allocation_size = 0;
602 for (
const AllocationHandle &handle : allocations_) {
603 total_allocation_size += handle.allocation_info.size;
605 float ratio =
float(current_usage_data_.acquired_segment_size_max) /
606 float(total_allocation_size);
608 std::string log_message = fmt::format(
"VKTexturePool uses {}/{} mb ({:.1f}%% of {} allocations)",
609 current_usage_data_.acquired_segment_size_max >> 20,
610 total_allocation_size >> 20,
612 current_usage_data_.allocation_count);
614 log_message += fmt::format(
" ({} cached VkImages)", current_usage_data_.image_cache_size);
#define CLOG_TRACE(clg_ref,...)
#define BLI_assert_unreachable()
#define UNUSED_VARS_NDEBUG(...)
#define BLI_assert_msg(a, msg)
render_graph::VKResourceStateTracker resources
VmaAllocator mem_allocator_get() const
VkDevice vk_handle() const
const VKExtensions & extensions_get() const
VolkDeviceTable functions
static VKDiscardPool & discard_pool_get()
void reset_unused_cycles_count(const VKImageInfo &info)
VKResourceWithHandle< VkImage > get_or_create(const VKImageInfo &info)
Vector< VKImageView > take_image_views(const VKImageInfo &info)
void store_image_views(const VKImageInfo &info, Vector< VKImageView > image_views)
void reset(bool force_reset=false)
void discard_all_of(VmaAllocation allocation)
Texture * acquire_texture_impl(int3 extent, int mip_len, GPUTextureType type, TextureFormat format, eGPUTextureUsage usage=GPU_TEXTURE_USAGE_GENERAL, const char *name=nullptr) override
void release_texture(Texture *texture) override
void reset(bool force_free=false) override
void offset_users_count(Texture *texture, int offset) override
TextureFormat device_format_get() const
VkImage vk_image_handle() const
VKTexture(const char *name)
VKSynchronizationCreateInfo CreateInfo
ResourceHandle add_aliased_image(VkImage vk_image, bool use_subresource_tracking, VmaAllocation allocation=VK_NULL_HANDLE, const char *name=nullptr)
Register an image resource that can have aliased memory.
void deref_resource(ResourceHandle resource_handle)
TEX_TEMPLATE DataVec texture(T, FltCoord, float=0.0f) RET
Segment< FEdge *, Vec3r > segment
constexpr auto tie(const VkImageCreateInfo &info)
void object_label(GLenum type, GLuint object, const char *name)
static VKResourceWithHandle< VkImage > create_and_bind_vk_image(const VKImageInfo &info, const std::string &name_str)
VkImageUsageFlags to_vk_image_usage(const eGPUTextureUsage usage, const GPUTextureFormatFlag format_flag, bool use_image_host_copy)
VkImageType to_vk_image_type(const GPUTextureType type)
VkFormat to_vk_format(const TextureFormat format)
VkImageCreateFlags to_vk_image_create(const GPUTextureType texture_type, const GPUTextureFormatFlag format_flag, eGPUTextureUsage usage)
@ GPU_FORMAT_DEPTH_STENCIL
BLI_INLINE MTLTexture * unwrap(Texture *tex)
VkImageAspectFlags to_vk_image_aspect_flag_bits(const TextureFormat format)
BLI_INLINE Texture * wrap(MTLTexture *texture)
VkMemoryRequirements get_image_memory_requirements(const VkImageCreateInfo &image_info)
GPUTextureFormatFlag to_format_flag(TextureFormat format)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
unsigned __int64 uint64_t
@ GPU_SAMPLER_FILTERING_LINEAR
MINLINE int max_ii(int a, int b)
bool GPU_type_matches(GPUDeviceType device, GPUOSType os, GPUDriverType driver)
VecBase< int32_t, 3 > int3
MINLINE uint64_t ceil_to_multiple_ul(uint64_t a, uint64_t b)
MINLINE int min_ii(int a, int b)
constexpr uint64_t get_default_hash(const T &v, const Args &...args)
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
constexpr uint64_t operator()(const VkImageCreateInfo &value) const
VkImageCreateInfo create_info
bool operator==(const VKImageInfo &) const
ResourceHandle resource_handle