16#include "util/image.h"
23#include <OpenImageIO/thread.h>
55 const size_t clipped_end = std::min(end,
evicted_mask.size());
56 for (
size_t i = begin; i < clipped_end; i++) {
97ImageCache::DeviceImageKey ImageCache::DeviceImage::key()
const
116 images_first_free.clear();
136 image_info_id = images.
size();
139 &device,
"full_image", image_info_id, type, interpolation, extension);
142 img->alloc(width, height);
147 deferred_updates.insert(&mem);
152void ImageCache::free_full(
const uint image_info_id)
155 deferred_updates.erase(images[image_info_id]);
156 images.steal(image_info_id);
162 free_tiled_image(dscene, tex);
178 for (
int i = 0; i < tex.
tile_num; i++) {
180 free_tile(descriptors[i]);
189 stats.clear_range(begin, begin +
size_t(tex.
tile_num));
191template<TypeDesc::BASETYPE FileFormat,
typename StorageType>
192device_image *ImageCache::load_full(Device &device,
194 const ImageMetaData &metadata,
197 const float texture_resolution,
206 const int width = metadata.
width;
207 const int height = metadata.
height;
210 vector<StorageType> pixels_storage;
219 float scale_factor = 1.0f;
220 if (texture_resolution < 1.0f) {
221 scale_factor =
powf(2.0f,
floorf(log2f(texture_resolution)));
226 if (scale_factor > 0.0f && scale_factor < 1.0f) {
227 pixels_storage.resize(
int64_t(width) * height * 4);
228 pixels = &pixels_storage[0];
233 device, metadata.
type, interpolation, extension, width, height, image_info_id);
234 pixels = mem->
data<StorageType>();
237 if (pixels ==
nullptr || !loader.
load_pixels(metadata, pixels)) {
240 free_full(image_info_id);
247 if (!pixels_storage.empty()) {
248 LOG_DEBUG <<
"Scaling image " << loader.
name() <<
" by a factor of " << scale_factor <<
".";
249 vector<StorageType> scaled_pixels;
262 mem = &alloc_full(device,
269 StorageType *texture_pixels = mem->
data<StorageType>();
270 std::copy_n(scaled_pixels.data(), scaled_pixels.size(), texture_pixels);
279 const float texture_resolution,
291 mem = load_full<TypeDesc::FLOAT, float>(
292 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
295 mem = load_full<TypeDesc::FLOAT, float>(
296 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
299 mem = load_full<TypeDesc::UINT8, uchar>(
300 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
303 mem = load_full<TypeDesc::UINT8, uchar>(
304 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
307 mem = load_full<TypeDesc::HALF, half>(
308 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
311 mem = load_full<TypeDesc::HALF, half>(
312 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
315 mem = load_full<TypeDesc::USHORT, uint16_t>(
316 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
319 mem = load_full<TypeDesc::USHORT, uint16_t>(
320 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
330 device, type, interpolation, extension, metadata.
nanovdb_byte_size, 0, image_info_id);
332 uint8_t *pixels = mem->
data<uint8_t>();
353 const int tile_size_padded,
358 DeviceImage *img =
nullptr;
359 int tile_offset = -1;
362 const DeviceImageKey key = {
363 .type = type, .interpolation = interpolation, .tile_size = tile_size_padded};
365 size_t first_free = 0;
366 auto it = images_first_free.find(key);
367 if (it != images_first_free.end()) {
368 first_free = it->second;
371 for (
size_t i = first_free; i < images.size(); i++) {
372 DeviceImage *img_candidate = images[i];
373 if (img_candidate && img_candidate->occupancy != ~
uint64_t(0) && img_candidate->key() == key) {
377 tile_offset =
bitscan(~img->occupancy);
382 const bool alloc_image = img ==
nullptr;
386 for (image_info_id = 0; image_info_id < images.size(); image_info_id++) {
387 if (!images[image_info_id]) {
391 if (image_info_id == images.size()) {
392 images.resize(images.size() + 1);
396 unique_ptr<DeviceImage> new_img = make_unique<DeviceImage>(
397 &device,
"tile_image", image_info_id, type, interpolation,
EXTENSION_EXTEND);
399 images.replace(image_info_id, std::move(new_img));
401 img->alloc(tile_size_padded * TILE_IMAGE_MAX_TILES, tile_size_padded);
404 stats.add_tiled_bytes(img->memory_size());
406 auto it_first_free = images_first_free.find(key);
407 if (it_first_free == images_first_free.end()) {
408 images_first_free[key] = size_t(image_info_id);
411 it_first_free->second = std::min(
size_t(image_info_id), it_first_free->second);
422 img->copy_to_device();
423 deferred_updates.erase(img);
431 img->occupancy |= (
uint64_t(1) << tile_offset);
434 if (img->occupancy == ~
uint64_t(0)) {
435 size_t new_first_free = img->image_info_id + 1;
436 while (new_first_free < images.size()) {
437 DeviceImage *next = images[new_first_free];
438 if (next && next->occupancy != ~
uint64_t(0) && next->key() == key) {
443 images_first_free[key] = new_first_free;
457 DeviceImage *img = images[image_info_id];
458 assert(img && img->image_info_id == image_info_id);
460 img->occupancy &= ~(
uint64_t(1) << tile_offset);
463 const DeviceImageKey key = img->key();
465 auto it_first_free = images_first_free.find(key);
466 if (it_first_free == images_first_free.end()) {
467 images_first_free[key] = size_t(image_info_id);
470 it_first_free->second = std::min(
size_t(image_info_id), it_first_free->second);
473 if (img->occupancy == 0 && free_empty_image) {
475 stats.remove_tiled_bytes(img->memory_size());
476 deferred_updates.erase(images[image_info_id]);
477 images.replace(image_info_id,
nullptr);
479 if (image_info_id == images_first_free[key]) {
481 size_t new_first_free = image_info_id + 1;
482 while (new_first_free < images.size()) {
483 DeviceImage *next = images[new_first_free];
484 if (next && next->occupancy != ~
uint64_t(0) && next->key() == key) {
489 images_first_free[key] = new_first_free;
503 const int tile_size = metadata.
tile_size;
510 for (
int miplevel = 0; max_miplevels; miplevel++) {
511 const int mip_width = std::max(1, tex.
width >> miplevel);
512 const int mip_height = std::max(1, tex.
height >> miplevel);
517 if (mip_width <= tile_size && mip_height <= tile_size) {
530 std::span<KernelImageTexture> image_textures)
533 size_t num_descriptors = 0;
540 if (num_descriptors == 0) {
550 size_t offset = tile_descriptors.
size();
551 tile_descriptors.
resize(offset + num_descriptors);
556 "KERNEL_TILE_LOAD_NONE must be all 0xFF bytes for memset initialization");
559 const size_t tile_access_old_size = tile_access.
size();
560 if (tile_descriptors.
size() > tile_access_old_size) {
562 memset(tile_access.
data() + tile_access_old_size,
564 tile_descriptors.
size() - tile_access_old_size);
567 stats.resize(tile_descriptors.
size());
581 for (
int miplevel = 0; miplevel < tex.
tile_levels; miplevel++) {
584 const int mip_width = std::max(1, tex.
width >> miplevel);
585 const int mip_height = std::max(1, tex.
height >> miplevel);
597void ImageCache::load_tile(
Device &device,
606 const size_t bit_index,
609 const int width = std::max(
int64_t(1), metadata.
width >> miplevel);
610 const int height = std::max(
int64_t(1), metadata.
height >> miplevel);
611 const int tile_size = metadata.
tile_size;
612 const size_t w =
min(
size_t(width -
x),
size_t(tile_size));
613 const size_t h =
min(
size_t(height -
y),
size_t(tile_size));
619 device, metadata.
type, interpolation, tile_size_padded, tile_descriptor);
622 const size_t x_stride = pixel_bytes;
623 const size_t y_stride = mem.
data_width * pixel_bytes;
627 uint8_t *pixels = mem.
data<uint8_t>() + x_offset;
649 LOG_TRACE <<
"Load image tile: " << loader.
name() <<
", mip level " << miplevel <<
" (" <<
x
653 LOG_WARNING <<
"Failed to load image tile: " << loader.
name() <<
", mip level " << miplevel
654 <<
" (" <<
x <<
" " <<
y <<
")";
666 free_tile(tile_descriptor,
false);
672 free_tile(tile_descriptor,
false);
677 stats.load_tile(bit_index);
680 deferred_updates.insert(&mem);
690 const int tile_levels,
691 const size_t tile_idx,
692 size_t &r_level_start)
695 size_t level_start = 0;
696 for (
int m = 0; m < tile_levels; m++) {
697 const size_t level_offset = levels[m] - tile_levels;
698 if (tile_idx < level_offset) {
701 level_start = level_offset;
704 r_level_start = level_start;
713 const int miplevel_offset,
714 const uint8_t *access_state)
716 const int tile_size = metadata.
tile_size;
726 for (
size_t tile_idx = 0; tile_idx < tex.
tile_num; tile_idx++) {
749 const size_t idx_in_level = tile_idx - level_start;
750 const int mip_width = std::max(1, tex.
width >> miplevel);
751 const size_t tiles_x =
divide_up(mip_width, tile_size);
752 const size_t tile_y = idx_in_level / tiles_x;
753 const size_t tile_x = idx_in_level % tiles_x;
754 const size_t x = tile_x * tile_size;
755 const size_t y = tile_y * tile_size;
763 miplevel + miplevel_offset,
766 base_offset + tile_idx,
767 descriptors[tile_idx]);
780 const int miplevel_offset)
799 miplevel + miplevel_offset,
811 OIIO::atomic_backoff backoff;
834 const int tile_size = metadata.
tile_size;
836 const size_t tile_bytes = tile_size * tile_size * pixel_bytes;
838 for (
int miplevel = 0; miplevel < tex.
tile_levels; miplevel++) {
839 const int width = std::max(1, tex.
width >> miplevel);
840 const int height = std::max(1, tex.
height >> miplevel);
841 const int tiles_x =
divide_up(width, tile_size);
842 const int tiles_y =
divide_up(height, tile_size);
843 const int tiles_total = tiles_x * tiles_y;
846 const size_t level_start = levels[miplevel] - tex.
tile_levels;
847 int tiles_loaded = 0;
848 for (
int i = 0; i < tiles_total; i++) {
855 mip_stats.
width = width;
856 mip_stats.
height = height;
861 tile_stats.
size += tiles_loaded * tile_bytes;
867 std::span<KernelImageTexture> image_textures,
868 const uint8_t *access_state)
871 if (tile_descriptors.
size() == 0) {
876 const size_t preserve_budget = size_t(
DebugFlags().texture_cache.preserve_unused) * 1024 * 1024;
889 size_t num_evicted = 0;
890 size_t num_preserved = 0;
891 size_t preserved_bytes = 0;
894 const auto evict_tile = [&](
const size_t global_idx) {
896 free_tile(descriptor);
899 stats.evict_tile(global_idx);
902 for (
size_t img_idx = 0; img_idx < image_textures.size(); img_idx++) {
909 for (
int i = 0; i < tex.
tile_num; i++) {
910 const size_t global_idx = base_offset + i;
922 DeviceImage *img = images[image_info_id];
923 bool is_host_mapped =
false;
924 size_t tile_bytes = 0;
929 tile_bytes = tile_size_padded * tile_size_padded * pixel_bytes;
932 if (is_host_mapped) {
934 evict_tile(global_idx);
937 unused_tiles.push_back({global_idx, tile_bytes});
938 preserved_bytes += tile_bytes;
946 for (
const UnusedTile &
tile : unused_tiles) {
947 if (preserved_bytes <= preserve_budget) {
953 evict_tile(
tile.global_idx);
954 preserved_bytes -=
tile.tile_bytes;
958 if (num_evicted > 0) {
960 LOG_DEBUG <<
"Texture cache tile eviction: " << num_evicted <<
" evicted, " << num_used
961 <<
" used, " << num_preserved <<
" preserved (" << preserved_bytes / (1024 * 1024)
979 copy_images_to_device_no_lock();
998 copy_images_to_device_no_lock();
1011void ImageCache::copy_images_to_device_no_lock()
1016 deferred_updates.clear();
ATOMIC_INLINE uint32_t atomic_load_uint32(const uint32_t *v)
ATOMIC_INLINE void atomic_store_uint32(uint32_t *p, uint32_t v)
ATOMIC_INLINE uint32_t atomic_cas_uint32(uint32_t *v, uint32_t old, uint32_t _new)
unsigned long long int uint64_t
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
virtual void load_image_info()=0
virtual void copy_to_device(device_memory &mem)=0
device_vector< uint8_t > image_texture_tile_access_state
device_vector< KernelTileDescriptor > image_texture_tile_descriptors
virtual bool has_unified_image_memory_all() const
void load_requested_tiles(Device &device, DeviceScene &dscene, const KernelImageTexture &tex, ImageLoader &loader, const ImageMetaData &metadata, int miplevel_offset, const uint8_t *access_state)
void device_free(DeviceScene &dscene)
void load_image_tiled_descriptors(DeviceScene &dscene, std::span< KernelImageTexture > image_textures)
void load_image_tiled(const ImageMetaData &metadata, KernelImageTexture &tex)
void free_image(DeviceScene &dscene, const KernelImageTexture &tex)
void load_requested_tile(Device &device, DeviceScene &dscene, const KernelImageTexture &tex, KernelTileDescriptor &r_tile_descriptor, int miplevel, int x, int y, ImageLoader &loader, const ImageMetaData &metadata, int miplevel_offset)
void collect_statistics(DeviceScene &dscene, const KernelImageTexture &tex, const ImageMetaData &metadata, ImageTileStats &tile_stats)
void evict_unused(const Device &device, DeviceScene &dscene, std::span< KernelImageTexture > image_textures, const uint8_t *access_state)
device_image * load_image_full(Device &device, ImageLoader &loader, const ImageMetaData &metadata, const float texture_resolution, KernelImageTexture &tex)
size_t memory_size(DeviceScene &dscene) const
void copy_to_device(DeviceScene &dscene)
virtual bool load_pixels_tile(const ImageMetaData &, const int, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const ExtensionType, uint8_t *)
virtual bool load_pixels(const ImageMetaData &metadata, void *pixels)=0
virtual string name() const =0
void copy_to_device_if_modified()
T * resize(const size_t width, const size_t height=0)
void push_back(unique_ptr< T > &&value)
void util_image_resize_pixels(const vector< T > &input_pixels, const int64_t input_width, const int64_t input_height, const int64_t components, vector< T > *output_pixels, int64_t *output_width, int64_t *output_height)
#define CCL_NAMESPACE_END
#define assert(assertion)
static int image_tile_find_miplevel(const KernelTileDescriptor *levels, const int tile_levels, const size_t tile_idx, size_t &r_level_start)
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int num_tiles
const ccl_global KernelWorkTile * tile
#define KERNEL_TILE_LOAD_NONE
#define KERNEL_TILE_LOAD_FAILED
#define KERNEL_TILE_LOAD_REQUEST
#define KERNEL_IMAGE_TEX_PADDING
#define KERNEL_IMAGE_NONE
#define KERNEL_TILE_ACCESS_USED
#define KERNEL_TILE_ACCESS_NONE
#define KERNEL_TILE_ACCESS_REQUESTED
__forceinline uint32_t bitscan(const uint32_t value)
__forceinline uint32_t __bsr(const uint32_t x)
void resize(const size_t size)
int64_t current_tiled_bytes
void remove_tiled_bytes(const size_t bytes)
void load_tile(const size_t bit_index)
void evict_tile(const size_t bit_index)
vector< uint8_t > evicted_mask
void add_tiled_bytes(const size_t bytes)
void clear_range(const size_t begin, const size_t end)
vector< ImageMipLevelStats > mip_levels
uint tile_descriptor_offset
std::unique_lock< std::mutex > thread_scoped_lock
ccl_device_inline bool is_power_of_two(const size_t x)
ccl_device_inline size_t divide_up(const size_t x, const size_t y)
uint KernelTileDescriptor
@ IMAGE_DATA_TYPE_NANOVDB_FP16
@ IMAGE_DATA_TYPE_USHORT4
@ IMAGE_DATA_TYPE_NANOVDB_EMPTY
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT3
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT4
@ IMAGE_DATA_TYPE_NANOVDB_FPN
ccl_device_inline bool kernel_tile_descriptor_loaded(const KernelTileDescriptor tile)
ccl_device_inline KernelTileDescriptor kernel_tile_descriptor_encode(const uint image_info_id, const uint offset)
ccl_device_inline uint kernel_tile_descriptor_offset(const KernelTileDescriptor tile)
ccl_device_inline uint kernel_tile_descriptor_image_info_id(const KernelTileDescriptor tile)