Blender V5.3
image_cache.cpp
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011-2025 Blender Foundation
2 *
3 * SPDX-License-Identifier: Apache-2.0 */
4
5#include "scene/image_cache.h"
6
7#include "device/device.h"
8#include "device/queue.h"
9
10#include "scene/devicescene.h"
11#include "scene/image_loader.h"
12#include "scene/stats.h"
13
14#include "util/atomic.h"
15#include "util/debug.h"
16#include "util/image.h"
17#include "util/image_impl.h"
18#include "util/image_metadata.h"
19#include "util/log.h"
20#include "util/simd.h"
21#include "util/tbb.h"
22
23#include <OpenImageIO/thread.h>
24
25#include <algorithm>
26
28
29/* ImageCacheStats. */
30
32{
34 evicted_mask.clear();
37 total_loaded = 0;
38 total_evicted = 0;
40 peak_loaded = 0;
42}
43
44void ImageCacheStats::resize(const size_t size)
45{
47 if (size > evicted_mask.size()) {
48 evicted_mask.resize(size, 0);
49 }
50}
51
52void ImageCacheStats::clear_range(const size_t begin, const size_t end)
53{
55 const size_t clipped_end = std::min(end, evicted_mask.size());
56 for (size_t i = begin; i < clipped_end; i++) {
57 evicted_mask[i] = 0;
58 }
59}
60
61void ImageCacheStats::load_tile(const size_t bit_index)
62{
67
68 if (bit_index < evicted_mask.size() && evicted_mask[bit_index] != 0) {
69 evicted_mask[bit_index] = 0;
71 }
72}
73
74void ImageCacheStats::evict_tile(const size_t bit_index)
75{
77 evicted_mask[bit_index] = 1;
80}
81
88
90{
92 current_tiled_bytes -= bytes;
93}
94
95/* ImageCache::DeviceImage */
96
97ImageCache::DeviceImageKey ImageCache::DeviceImage::key() const
98{
99 return {.type = ImageDataType(info.data_type),
100 .interpolation = InterpolationType(info.interpolation),
101 .tile_size = int(info.height)};
102}
103
104/* ImageCache */
105
106ImageCache::ImageCache() = default;
107
109{
110 assert(images.empty());
111}
112
114{
115 images.clear();
116 images_first_free.clear();
119
120 /* Reset eviction statistics. */
121 stats.reset();
122}
123
124/* Full image management. */
125
126device_image &ImageCache::alloc_full(Device &device,
127 ImageDataType type,
128 InterpolationType interpolation,
129 ExtensionType extension,
130 const int64_t width,
131 const int64_t height,
132 uint &image_info_id)
133{
134 thread_scoped_lock device_lock(device_mutex);
135
136 image_info_id = images.size();
137
138 unique_ptr<DeviceImage> img = make_unique<DeviceImage>(
139 &device, "full_image", image_info_id, type, interpolation, extension);
140
141 img->occupancy = ~uint64_t(0);
142 img->alloc(width, height);
143
144 images.push_back(std::move(img));
145
146 device_image &mem = *images.back();
147 deferred_updates.insert(&mem);
148
149 return mem;
150}
151
152void ImageCache::free_full(const uint image_info_id)
153{
154 thread_scoped_lock device_lock(device_mutex);
155 deferred_updates.erase(images[image_info_id]);
156 images.steal(image_info_id);
157}
158
160{
162 free_tiled_image(dscene, tex);
163 }
164 else if (tex.image_info_id != KERNEL_IMAGE_NONE) {
165 free_full(tex.image_info_id);
166 }
167}
168
169void ImageCache::free_tiled_image(DeviceScene &dscene, const KernelImageTexture &tex)
170{
171 /* Hold the mutex across the whole loop as tile_descriptors may get resized elsewhere. */
172 thread_scoped_lock device_lock(device_mutex);
173
174 /* TODO: Shrink tile_descriptors by compacting. */
177
178 for (int i = 0; i < tex.tile_num; i++) {
179 if (kernel_tile_descriptor_loaded(descriptors[i])) {
180 free_tile(descriptors[i]);
181 }
182 descriptors[i] = KERNEL_TILE_LOAD_NONE;
183 }
184
186
187 /* Clear eviction statistics bits for this image's tile descriptor range. */
188 const size_t begin = size_t(tex.tile_descriptor_offset) + size_t(tex.tile_levels);
189 stats.clear_range(begin, begin + size_t(tex.tile_num));
190}
191template<TypeDesc::BASETYPE FileFormat, typename StorageType>
192device_image *ImageCache::load_full(Device &device,
193 ImageLoader &loader,
194 const ImageMetaData &metadata,
195 const InterpolationType interpolation,
196 const ExtensionType extension,
197 const float texture_resolution,
198 uint &image_info_id)
199{
200 /* Ignore empty images. */
201 if (!(metadata.channels > 0)) {
202 return nullptr;
203 }
204
205 /* Get metadata. */
206 const int width = metadata.width;
207 const int height = metadata.height;
208
209 /* Read pixels. */
210 vector<StorageType> pixels_storage;
211 StorageType *pixels;
212 const int64_t max_size = max(width, height);
213 if (max_size == 0) {
214 /* Don't bother with empty images. */
215 return nullptr;
216 }
217
218 /* Compute scale factor rounded down to nearest power of 2. */
219 float scale_factor = 1.0f;
220 if (texture_resolution < 1.0f) {
221 scale_factor = powf(2.0f, floorf(log2f(texture_resolution)));
222 }
223
224 /* Allocate memory as needed, may be smaller to resize down. */
225 device_image *mem;
226 if (scale_factor > 0.0f && scale_factor < 1.0f) {
227 pixels_storage.resize(int64_t(width) * height * 4);
228 pixels = &pixels_storage[0];
229 mem = nullptr;
230 }
231 else {
232 mem = &alloc_full(
233 device, metadata.type, interpolation, extension, width, height, image_info_id);
234 pixels = mem->data<StorageType>();
235 }
236
237 if (pixels == nullptr || !loader.load_pixels(metadata, pixels)) {
238 /* Out of memory or failed to load image. */
239 if (mem) {
240 free_full(image_info_id);
241 image_info_id = KERNEL_IMAGE_NONE;
242 }
243 return nullptr;
244 }
245
246 /* Scale image down if needed. */
247 if (!pixels_storage.empty()) {
248 LOG_DEBUG << "Scaling image " << loader.name() << " by a factor of " << scale_factor << ".";
249 vector<StorageType> scaled_pixels;
250 int64_t scaled_width;
251 int64_t scaled_height;
252
253 util_image_resize_pixels(pixels_storage,
254 width,
255 height,
256 metadata.is_rgba() ? 4 : 1,
257 scale_factor,
258 &scaled_pixels,
259 &scaled_width,
260 &scaled_height);
261
262 mem = &alloc_full(device,
263 metadata.type,
264 interpolation,
265 extension,
266 scaled_width,
267 scaled_height,
268 image_info_id);
269 StorageType *texture_pixels = mem->data<StorageType>();
270 std::copy_n(scaled_pixels.data(), scaled_pixels.size(), texture_pixels);
271 }
272
273 return mem;
274}
275
277 ImageLoader &loader,
278 const ImageMetaData &metadata,
279 const float texture_resolution,
281{
282 const ImageDataType type = metadata.type;
283 const InterpolationType interpolation = InterpolationType(tex.interpolation);
284 const ExtensionType extension = ExtensionType(tex.extension);
285 device_image *mem = nullptr;
286 uint image_info_id = KERNEL_IMAGE_NONE;
287
288 /* Create new texture. */
289 switch (type) {
291 mem = load_full<TypeDesc::FLOAT, float>(
292 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
293 break;
295 mem = load_full<TypeDesc::FLOAT, float>(
296 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
297 break;
299 mem = load_full<TypeDesc::UINT8, uchar>(
300 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
301 break;
303 mem = load_full<TypeDesc::UINT8, uchar>(
304 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
305 break;
307 mem = load_full<TypeDesc::HALF, half>(
308 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
309 break;
311 mem = load_full<TypeDesc::HALF, half>(
312 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
313 break;
315 mem = load_full<TypeDesc::USHORT, uint16_t>(
316 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
317 break;
319 mem = load_full<TypeDesc::USHORT, uint16_t>(
320 device, loader, metadata, interpolation, extension, texture_resolution, image_info_id);
321 break;
328#ifdef WITH_NANOVDB
329 mem = &alloc_full(
330 device, type, interpolation, extension, metadata.nanovdb_byte_size, 0, image_info_id);
331
332 uint8_t *pixels = mem->data<uint8_t>();
333 if (pixels) {
334 loader.load_pixels(metadata, pixels);
335 }
336#endif
337 break;
338 }
340 break;
341 }
342
343 tex.image_info_id = image_info_id;
344
345 return mem;
346}
347
348/* Tile image pool management. */
349
350device_image &ImageCache::alloc_tile(Device &device,
351 ImageDataType type,
352 InterpolationType interpolation,
353 const int tile_size_padded,
354 KernelTileDescriptor &r_tile_descriptor)
355{
356 thread_scoped_lock device_lock(device_mutex);
357
358 DeviceImage *img = nullptr;
359 int tile_offset = -1;
360
361 /* Find image with free space by iterating pooled images. */
362 const DeviceImageKey key = {
363 .type = type, .interpolation = interpolation, .tile_size = tile_size_padded};
364
365 size_t first_free = 0;
366 auto it = images_first_free.find(key);
367 if (it != images_first_free.end()) {
368 first_free = it->second;
369 }
370
371 for (size_t i = first_free; i < images.size(); i++) {
372 DeviceImage *img_candidate = images[i];
373 if (img_candidate && img_candidate->occupancy != ~uint64_t(0) && img_candidate->key() == key) {
374 img = img_candidate;
375
376 /* Find unoccupied space in image. */
377 tile_offset = bitscan(~img->occupancy);
378 break;
379 }
380 }
381
382 const bool alloc_image = img == nullptr;
383 if (alloc_image) {
384 /* Allocate new image. */
385 uint image_info_id;
386 for (image_info_id = 0; image_info_id < images.size(); image_info_id++) {
387 if (!images[image_info_id]) {
388 break;
389 }
390 }
391 if (image_info_id == images.size()) {
392 images.resize(images.size() + 1);
393 }
394
395 /* Extension doesn't matter as we do it through padding. */
396 unique_ptr<DeviceImage> new_img = make_unique<DeviceImage>(
397 &device, "tile_image", image_info_id, type, interpolation, EXTENSION_EXTEND);
398 img = new_img.get();
399 images.replace(image_info_id, std::move(new_img));
400
401 img->alloc(tile_size_padded * TILE_IMAGE_MAX_TILES, tile_size_padded);
402 tile_offset = 0;
403
404 stats.add_tiled_bytes(img->memory_size());
405
406 auto it_first_free = images_first_free.find(key);
407 if (it_first_free == images_first_free.end()) {
408 images_first_free[key] = size_t(image_info_id);
409 }
410 else {
411 it_first_free->second = std::min(size_t(image_info_id), it_first_free->second);
412 }
413 }
414
415 if (alloc_image) {
416 /* Allocate the new image on all devices immediately:
417 * - If one of the devices is CPU or Metal using unified memory, the tile
418 * descriptor will be updated and rendering kernels can start using the new
419 * image immediately.
420 * - Allocation may perform host memory fallback, and other GPU device
421 * threads should not perform copies to an old host pointer. */
422 img->copy_to_device();
423 deferred_updates.erase(img);
424 }
425
426 /* Note: deferred update insertion is delayed until after the tile pixels
427 * have been copied, so another device will not finalize its update before
428 * pixels are in device memory. */
429
430 /* Mark tile as occupied and compute descriptor. */
431 img->occupancy |= (uint64_t(1) << tile_offset);
432
433 /* Maintain images_first_free index for this key. */
434 if (img->occupancy == ~uint64_t(0)) {
435 size_t new_first_free = img->image_info_id + 1;
436 while (new_first_free < images.size()) {
437 DeviceImage *next = images[new_first_free];
438 if (next && next->occupancy != ~uint64_t(0) && next->key() == key) {
439 break;
440 }
441 new_first_free++;
442 }
443 images_first_free[key] = new_first_free;
444 }
445
446 r_tile_descriptor = kernel_tile_descriptor_encode(img->image_info_id, tile_offset);
447
448 return *img;
449}
450
451void ImageCache::free_tile(const KernelTileDescriptor tile, const bool free_empty_image)
452{
453 const uint image_info_id = kernel_tile_descriptor_image_info_id(tile);
454 const uint tile_offset = kernel_tile_descriptor_offset(tile);
455
456 /* Look up pooled image by image_info_id. */
457 DeviceImage *img = images[image_info_id];
458 assert(img && img->image_info_id == image_info_id);
459
460 img->occupancy &= ~(uint64_t(1) << tile_offset);
461
462 /* Reconstruct key to update first_free map. */
463 const DeviceImageKey key = img->key();
464
465 auto it_first_free = images_first_free.find(key);
466 if (it_first_free == images_first_free.end()) {
467 images_first_free[key] = size_t(image_info_id);
468 }
469 else {
470 it_first_free->second = std::min(size_t(image_info_id), it_first_free->second);
471 }
472
473 if (img->occupancy == 0 && free_empty_image) {
474 /* All tiles free, remove the device image entirely. */
475 stats.remove_tiled_bytes(img->memory_size());
476 deferred_updates.erase(images[image_info_id]);
477 images.replace(image_info_id, nullptr);
478
479 if (image_info_id == images_first_free[key]) {
480 /* Search for next free one. */
481 size_t new_first_free = image_info_id + 1;
482 while (new_first_free < images.size()) {
483 DeviceImage *next = images[new_first_free];
484 if (next && next->occupancy != ~uint64_t(0) && next->key() == key) {
485 break;
486 }
487 new_first_free++;
488 }
489 images_first_free[key] = new_first_free;
490 }
491 }
492}
493
494/* Tile descriptor management. */
495
497{
499
501 tex.tile_size_shift = __bsr(metadata.tile_size);
502
503 const int tile_size = metadata.tile_size;
504 const InterpolationType interpolation = InterpolationType(tex.interpolation);
505 const int max_miplevels = interpolation != INTERPOLATION_CLOSEST ? 1 : INT_MAX;
506
507 int num_levels = 0;
508 int num_tiles = 0;
509
510 for (int miplevel = 0; max_miplevels; miplevel++) {
511 const int mip_width = std::max(1, tex.width >> miplevel);
512 const int mip_height = std::max(1, tex.height >> miplevel);
513
514 num_levels++;
515 num_tiles += divide_up(mip_width, tile_size) * divide_up(mip_height, tile_size);
516
517 if (mip_width <= tile_size && mip_height <= tile_size) {
518 break;
519 }
520 }
521
522 /* The descriptor range is assigned later in #load_image_tiled_descriptors,
523 * to avoid lock contention. */
525 tex.tile_levels = num_levels;
526 tex.tile_num = num_tiles;
527}
528
530 std::span<KernelImageTexture> image_textures)
531{
532 /* Find which image textures need tile descriptors filled. */
533 size_t num_descriptors = 0;
534 for (const KernelImageTexture &tex : image_textures) {
536 num_descriptors += tex.tile_levels + tex.tile_num;
537 }
538 }
539
540 if (num_descriptors == 0) {
541 return;
542 }
543
544 /* Resize device vector. */
545 const thread_scoped_lock device_lock(device_mutex);
546
549
550 size_t offset = tile_descriptors.size();
551 tile_descriptors.resize(offset + num_descriptors);
552
553 /* Initialize with a single memset, with assumption about KERNEL_TILE_LOAD_NONE value.
554 * This is the bulk of memory that needs to be set. */
556 "KERNEL_TILE_LOAD_NONE must be all 0xFF bytes for memset initialization");
557 memset(tile_descriptors.data() + offset, 0xFF, num_descriptors * sizeof(KernelTileDescriptor));
558
559 const size_t tile_access_old_size = tile_access.size();
560 if (tile_descriptors.size() > tile_access_old_size) {
561 tile_access.resize(tile_descriptors.size());
562 memset(tile_access.data() + tile_access_old_size,
564 tile_descriptors.size() - tile_access_old_size);
565 }
566
567 stats.resize(tile_descriptors.size());
568
569 /* Fill mip level information in tile descriptors. */
570 for (KernelImageTexture &tex : image_textures) {
571 if (!(tex.tile_num > 0 && tex.tile_descriptor_offset == KERNEL_TILE_LOAD_NONE)) {
572 continue;
573 }
574
575 tex.tile_descriptor_offset = offset;
576
577 KernelTileDescriptor *descr_data = tile_descriptors.data() + offset;
578 const int tile_size = 1 << tex.tile_size_shift;
579 int num_tiles = 0;
580
581 for (int miplevel = 0; miplevel < tex.tile_levels; miplevel++) {
582 descr_data[miplevel] = tex.tile_levels + num_tiles;
583
584 const int mip_width = std::max(1, tex.width >> miplevel);
585 const int mip_height = std::max(1, tex.height >> miplevel);
586 num_tiles += divide_up(mip_width, tile_size) * divide_up(mip_height, tile_size);
587 }
588
589 assert(num_tiles == tex.tile_num);
590
591 offset += tex.tile_levels + tex.tile_num;
592 }
593}
594
595/* Tile request processing. */
596
597void ImageCache::load_tile(Device &device,
598 DeviceScene &dscene,
599 ImageLoader &loader,
600 const ImageMetaData &metadata,
601 const InterpolationType interpolation,
602 const ExtensionType extension,
603 const int miplevel,
604 const int x,
605 const int y,
606 const size_t bit_index,
607 KernelTileDescriptor &r_tile_descriptor)
608{
609 const int width = std::max(int64_t(1), metadata.width >> miplevel);
610 const int height = std::max(int64_t(1), metadata.height >> miplevel);
611 const int tile_size = metadata.tile_size;
612 const size_t w = min(size_t(width - x), size_t(tile_size));
613 const size_t h = min(size_t(height - y), size_t(tile_size));
614 const size_t tile_size_padded = tile_size + KERNEL_IMAGE_TEX_PADDING * 2;
615
616 KernelTileDescriptor tile_descriptor;
617
618 device_image &mem = alloc_tile(
619 device, metadata.type, interpolation, tile_size_padded, tile_descriptor);
620
621 const size_t pixel_bytes = mem.data_elements * datatype_size(mem.data_type);
622 const size_t x_stride = pixel_bytes;
623 const size_t y_stride = mem.data_width * pixel_bytes;
624 const size_t x_offset = kernel_tile_descriptor_offset(tile_descriptor) * tile_size_padded *
625 pixel_bytes;
626
627 uint8_t *pixels = mem.data<uint8_t>() + x_offset;
628
629 /* Isolate so we don't switch to a render task that waits for the same tile,
630 * in case (OIIO) loading does any parallel work. */
631 bool ok = false;
632 isolate_task([&]() {
633 ok = loader.load_pixels_tile(metadata,
634 miplevel,
635 x,
636 y,
637 w,
638 h,
639 x_stride,
640 y_stride,
642 extension,
643 pixels);
644 });
645
647
648 if (ok) {
649 LOG_TRACE << "Load image tile: " << loader.name() << ", mip level " << miplevel << " (" << x
650 << " " << y << ")";
651 }
652 else {
653 LOG_WARNING << "Failed to load image tile: " << loader.name() << ", mip level " << miplevel
654 << " (" << x << " " << y << ")";
655 }
656
657 /* Mark image for deferred GPU update, after pixels have been loaded to all devices.
658 * Insert GPU update and update tile descriptor under the same device lock.
659 *
660 * This is copied by the next GPU tile loading round, also for CPU cache misses.
661 * Copying a pooled tile image on every CPU cache miss is not needed. */
662 const thread_scoped_lock device_lock(device_mutex);
663
664 /* Another thread may have loaded the same tile concurrently, keep that one. */
665 if (kernel_tile_descriptor_loaded(r_tile_descriptor)) {
666 free_tile(tile_descriptor, false);
667 return;
668 }
669
670 if (!ok) {
671 /* Release the slot for another tile on failed to load. */
672 free_tile(tile_descriptor, false);
673 r_tile_descriptor = KERNEL_TILE_LOAD_FAILED;
674 return;
675 }
676
677 stats.load_tile(bit_index);
678
679 if (!device.has_unified_image_memory_all()) {
680 deferred_updates.insert(&mem);
681 }
682
683 /* Atomic store so render threads reading the descriptor without a lock
684 * see the pixels written above. */
685 atomic_store_uint32(&r_tile_descriptor, tile_descriptor);
686}
687
688/* Find the mip level that contains the tile index. */
690 const int tile_levels,
691 const size_t tile_idx,
692 size_t &r_level_start)
693{
694 int miplevel = 0;
695 size_t level_start = 0;
696 for (int m = 0; m < tile_levels; m++) {
697 const size_t level_offset = levels[m] - tile_levels;
698 if (tile_idx < level_offset) {
699 break;
700 }
701 level_start = level_offset;
702 miplevel = m;
703 }
704 r_level_start = level_start;
705 return miplevel;
706}
707
709 DeviceScene &dscene,
710 const KernelImageTexture &tex,
711 ImageLoader &loader,
712 const ImageMetaData &metadata,
713 const int miplevel_offset,
714 const uint8_t *access_state)
715{
716 const int tile_size = metadata.tile_size;
717 const InterpolationType interpolation = InterpolationType(tex.interpolation);
718 const ExtensionType extension = ExtensionType(tex.extension);
719 const size_t base_offset = tex.tile_descriptor_offset + tex.tile_levels;
720
721 KernelTileDescriptor *descriptors = dscene.image_texture_tile_descriptors.data() + base_offset;
724
725 /* Scan access state for this image's tiles. */
726 for (size_t tile_idx = 0; tile_idx < tex.tile_num; tile_idx++) {
727 if (!(access_state[base_offset + tile_idx] & KERNEL_TILE_ACCESS_REQUESTED)) {
728 continue;
729 }
730
731 /* Skip if tile is already loaded or failed. */
732 const KernelTileDescriptor existing = descriptors[tile_idx];
733 if (kernel_tile_descriptor_loaded(existing) || existing == KERNEL_TILE_LOAD_FAILED) {
734 continue;
735 }
736
737 /* Atomically claim this tile slot. If another thread or GPU callback wins the race,
738 * skip and let the winner load it. We don't require KERNEL_TILE_LOAD_REQUEST because
739 * the access state might be set without it even if that race condition is unlikely. */
741 &descriptors[tile_idx], KERNEL_TILE_LOAD_NONE, KERNEL_TILE_LOAD_REQUEST);
742 if (old != KERNEL_TILE_LOAD_NONE && old != KERNEL_TILE_LOAD_REQUEST) {
743 continue;
744 }
745
746 /* Find miplevel for this tile index, and compute tile pixel coordinates. */
747 size_t level_start;
748 const int miplevel = image_tile_find_miplevel(levels, tex.tile_levels, tile_idx, level_start);
749 const size_t idx_in_level = tile_idx - level_start;
750 const int mip_width = std::max(1, tex.width >> miplevel);
751 const size_t tiles_x = divide_up(mip_width, tile_size);
752 const size_t tile_y = idx_in_level / tiles_x;
753 const size_t tile_x = idx_in_level % tiles_x;
754 const size_t x = tile_x * tile_size;
755 const size_t y = tile_y * tile_size;
756
757 load_tile(device,
758 dscene,
759 loader,
760 metadata,
761 interpolation,
762 extension,
763 miplevel + miplevel_offset,
764 x,
765 y,
766 base_offset + tile_idx,
767 descriptors[tile_idx]);
768 }
769}
770
772 DeviceScene &dscene,
773 const KernelImageTexture &tex,
774 KernelTileDescriptor &r_tile_descriptor,
775 int miplevel,
776 int x,
777 int y,
778 ImageLoader &loader,
779 const ImageMetaData &metadata,
780 const int miplevel_offset)
781{
782 /* This is called by the CPU kernel to immediately load a tile. */
783
784 /* If we can atomically set KERNEL_TILE_LOAD_REQUEST, this thread is responsible
785 * for loading the tile. */
786 const KernelTileDescriptor tile_descriptor_old = atomic_cas_uint32(
788 if (tile_descriptor_old == KERNEL_TILE_LOAD_NONE) {
789 const InterpolationType interpolation = InterpolationType(tex.interpolation);
790 const ExtensionType extension = ExtensionType(tex.extension);
791 const size_t bit_index = &r_tile_descriptor - dscene.image_texture_tile_descriptors.data();
792
793 load_tile(device,
794 dscene,
795 loader,
796 metadata,
797 interpolation,
798 extension,
799 miplevel + miplevel_offset,
800 x,
801 y,
802 bit_index,
803 r_tile_descriptor);
804 return;
805 }
806 if (tile_descriptor_old != KERNEL_TILE_LOAD_REQUEST) {
807 return;
808 }
809
810 /* Wait for other thread to load the tile. */
811 OIIO::atomic_backoff backoff;
812 while (atomic_load_uint32(&r_tile_descriptor) == KERNEL_TILE_LOAD_REQUEST) {
813 backoff();
814 }
815}
816
817/* Statistics. */
818
820 const KernelImageTexture &tex,
821 const ImageMetaData &metadata,
822 ImageTileStats &tile_stats)
823{
825 return;
826 }
827
828 const KernelTileDescriptor *tile_descriptors = dscene.image_texture_tile_descriptors.data() +
832
833 /* Compute per-mip-level statistics. */
834 const int tile_size = metadata.tile_size;
835 const size_t pixel_bytes = metadata.pixel_memory_size();
836 const size_t tile_bytes = tile_size * tile_size * pixel_bytes;
837
838 for (int miplevel = 0; miplevel < tex.tile_levels; miplevel++) {
839 const int width = std::max(1, tex.width >> miplevel);
840 const int height = std::max(1, tex.height >> miplevel);
841 const int tiles_x = divide_up(width, tile_size);
842 const int tiles_y = divide_up(height, tile_size);
843 const int tiles_total = tiles_x * tiles_y;
844
845 /* Count loaded tiles for this mip level. */
846 const size_t level_start = levels[miplevel] - tex.tile_levels;
847 int tiles_loaded = 0;
848 for (int i = 0; i < tiles_total; i++) {
849 if (kernel_tile_descriptor_loaded(tile_descriptors[level_start + i])) {
850 tiles_loaded++;
851 }
852 }
853
854 ImageMipLevelStats mip_stats;
855 mip_stats.width = width;
856 mip_stats.height = height;
857 mip_stats.tiles_total = tiles_total;
858 mip_stats.tiles_loaded = tiles_loaded;
859 tile_stats.mip_levels.push_back(mip_stats);
860
861 tile_stats.size += tiles_loaded * tile_bytes;
862 }
863}
864
866 DeviceScene &dscene,
867 std::span<KernelImageTexture> image_textures,
868 const uint8_t *access_state)
869{
871 if (tile_descriptors.size() == 0) {
872 return;
873 }
874
875 const bool cpu_only = (device.info.type == DEVICE_CPU);
876 const size_t preserve_budget = size_t(DebugFlags().texture_cache.preserve_unused) * 1024 * 1024;
877
878 /* Hold the mutex for the entire eviction pass. */
879 thread_scoped_lock device_lock(device_mutex);
880
881 /* Collect unused tiles. */
882 struct UnusedTile {
883 size_t global_idx;
884 size_t tile_bytes;
885 };
886
887 vector<UnusedTile> unused_tiles;
888 size_t num_used = 0;
889 size_t num_evicted = 0;
890 size_t num_preserved = 0;
891 size_t preserved_bytes = 0;
892
893 /* Free a single unused tile, updating descriptor and statistics. */
894 const auto evict_tile = [&](const size_t global_idx) {
895 KernelTileDescriptor &descriptor = tile_descriptors[global_idx];
896 free_tile(descriptor);
897 descriptor = KERNEL_TILE_LOAD_NONE;
898 num_evicted++;
899 stats.evict_tile(global_idx);
900 };
901
902 for (size_t img_idx = 0; img_idx < image_textures.size(); img_idx++) {
903 const KernelImageTexture &tex = image_textures[img_idx];
905 continue;
906 }
907
908 const size_t base_offset = tex.tile_descriptor_offset + tex.tile_levels;
909 for (int i = 0; i < tex.tile_num; i++) {
910 const size_t global_idx = base_offset + i;
911 KernelTileDescriptor &descriptor = tile_descriptors[global_idx];
912 if (!kernel_tile_descriptor_loaded(descriptor)) {
913 continue;
914 }
915 if (access_state[global_idx] & KERNEL_TILE_ACCESS_USED) {
916 num_used++;
917 continue;
918 }
919
920 /* Look up the DeviceImage to determine host-mapped status and tile size. */
921 const uint image_info_id = kernel_tile_descriptor_image_info_id(descriptor);
922 DeviceImage *img = images[image_info_id];
923 bool is_host_mapped = false;
924 size_t tile_bytes = 0;
925 if (img) {
926 is_host_mapped = !cpu_only && (img->shared_pointer != nullptr);
927 const size_t pixel_bytes = img->data_elements * datatype_size(img->data_type);
928 const size_t tile_size_padded = img->data_height;
929 tile_bytes = tile_size_padded * tile_size_padded * pixel_bytes;
930 }
931
932 if (is_host_mapped) {
933 /* Host-mapped tiles are slower to sample, always evict. */
934 evict_tile(global_idx);
935 }
936 else {
937 unused_tiles.push_back({global_idx, tile_bytes});
938 preserved_bytes += tile_bytes;
939 }
940 }
941 }
942
943 /* Evict device-resident tiles until preserved bytes fits within the budget.
944 * Note this means tiles loaded earlier will be preserved, as tiles loaded later are
945 * less likely to be needed often. */
946 for (const UnusedTile &tile : unused_tiles) {
947 if (preserved_bytes <= preserve_budget) {
948 /* Within budget, preserve remaining device-resident tiles. */
949 num_preserved++;
950 }
951 else {
952 /* Over budget, evict to bring preserved bytes down. */
953 evict_tile(tile.global_idx);
954 preserved_bytes -= tile.tile_bytes;
955 }
956 }
957
958 if (num_evicted > 0) {
960 LOG_DEBUG << "Texture cache tile eviction: " << num_evicted << " evicted, " << num_used
961 << " used, " << num_preserved << " preserved (" << preserved_bytes / (1024 * 1024)
962 << " MB).";
963 }
964}
965
971
972/* Copy to device. */
973
975{
976 /* Copy all device memory managed by the image cache to all devices, for updates
977 * outside of rendering. */
978 thread_scoped_lock device_lock(device_mutex);
979 copy_images_to_device_no_lock();
980
983
986}
987
989{
990 /* Copy data for a single GPU device during rendering. Note this may run
991 * concurrently for multiple GPU devices, or CPU and GPU devices.
992 *
993 * We run copy_device_images which will allocate image memory on all devices,
994 * which is safe to do while kernels are executing because we only allocate
995 * without freeing, and load_image_info() is delayed. */
996 {
997 thread_scoped_lock device_lock(device_mutex);
998 copy_images_to_device_no_lock();
999
1000 /* Copy updated tile descriptors for this GPU device only.
1001 * Note the tile access buffer is not zeroed here as we want to keep the USED
1002 * state until cache eviction. Some tiles may remain as REQUESTED but that's
1003 * fine, they will be skipped in load_requested_tiles if already loaded. */
1005 }
1006
1007 /* Update image info only for this GPU device. */
1008 queue.load_image_info();
1009}
1010
1011void ImageCache::copy_images_to_device_no_lock()
1012{
1013 for (device_image *mem : deferred_updates) {
1014 mem->copy_to_device();
1015 }
1016 deferred_updates.clear();
1017}
1018
ATOMIC_INLINE uint32_t atomic_load_uint32(const uint32_t *v)
ATOMIC_INLINE void atomic_store_uint32(uint32_t *p, uint32_t v)
ATOMIC_INLINE uint32_t atomic_cas_uint32(uint32_t *v, uint32_t old, uint32_t _new)
volatile int lock
long long int int64_t
unsigned long long int uint64_t
btScalar max
SIMD_FORCE_INLINE const btScalar & w() const
Return the w value.
Definition btQuadWord.h:119
DeviceType type
virtual void load_image_info()=0
virtual void copy_to_device(device_memory &mem)=0
device_vector< uint8_t > image_texture_tile_access_state
device_vector< KernelTileDescriptor > image_texture_tile_descriptors
Definition devicescene.h:99
virtual bool has_unified_image_memory_all() const
DeviceInfo info
void load_requested_tiles(Device &device, DeviceScene &dscene, const KernelImageTexture &tex, ImageLoader &loader, const ImageMetaData &metadata, int miplevel_offset, const uint8_t *access_state)
void device_free(DeviceScene &dscene)
void load_image_tiled_descriptors(DeviceScene &dscene, std::span< KernelImageTexture > image_textures)
void load_image_tiled(const ImageMetaData &metadata, KernelImageTexture &tex)
void free_image(DeviceScene &dscene, const KernelImageTexture &tex)
void load_requested_tile(Device &device, DeviceScene &dscene, const KernelImageTexture &tex, KernelTileDescriptor &r_tile_descriptor, int miplevel, int x, int y, ImageLoader &loader, const ImageMetaData &metadata, int miplevel_offset)
void collect_statistics(DeviceScene &dscene, const KernelImageTexture &tex, const ImageMetaData &metadata, ImageTileStats &tile_stats)
void evict_unused(const Device &device, DeviceScene &dscene, std::span< KernelImageTexture > image_textures, const uint8_t *access_state)
device_image * load_image_full(Device &device, ImageLoader &loader, const ImageMetaData &metadata, const float texture_resolution, KernelImageTexture &tex)
size_t memory_size(DeviceScene &dscene) const
void copy_to_device(DeviceScene &dscene)
virtual bool load_pixels_tile(const ImageMetaData &, const int, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const ExtensionType, uint8_t *)
virtual bool load_pixels(const ImageMetaData &metadata, void *pixels)=0
virtual string name() const =0
ImageDataType type
bool is_rgba() const
int64_t nanovdb_byte_size
size_t pixel_memory_size() const
uint32_t tile_size
void copy_to_device()
Definition memory.cpp:313
KernelImageInfo info
size_t size() const
T * resize(const size_t width, const size_t height=0)
void push_back(unique_ptr< T > &&value)
size_t size() const
void util_image_resize_pixels(const vector< T > &input_pixels, const int64_t input_width, const int64_t input_height, const int64_t components, vector< T > *output_pixels, int64_t *output_width, int64_t *output_height)
#define powf(x, y)
#define CCL_NAMESPACE_END
@ DEVICE_CPU
#define assert(assertion)
#define INT_MAX
static int image_tile_find_miplevel(const KernelTileDescriptor *levels, const int tile_levels, const size_t tile_idx, size_t &r_level_start)
ccl_gpu_kernel_postfix ccl_global KernelWorkTile const int num_tiles
const ccl_global KernelWorkTile * tile
#define LOG_DEBUG
Definition log.h:107
#define LOG_WARNING
Definition log.h:103
#define LOG_TRACE
Definition log.h:108
#define KERNEL_TILE_LOAD_NONE
#define KERNEL_TILE_LOAD_FAILED
#define KERNEL_TILE_LOAD_REQUEST
#define KERNEL_IMAGE_TEX_PADDING
#define KERNEL_IMAGE_NONE
#define KERNEL_TILE_ACCESS_USED
#define KERNEL_TILE_ACCESS_NONE
#define KERNEL_TILE_ACCESS_REQUESTED
unsigned int uint
Definition BKE_text.h:55
iter begin(iter)
#define floorf
__forceinline uint32_t bitscan(const uint32_t value)
Definition simd.h:476
__forceinline uint32_t __bsr(const uint32_t x)
Definition simd.h:434
#define min(a, b)
Definition sort.cc:38
void resize(const size_t size)
int64_t current_tiled_bytes
Definition image_cache.h:34
void remove_tiled_bytes(const size_t bytes)
void load_tile(const size_t bit_index)
void evict_tile(const size_t bit_index)
int64_t peak_tiled_bytes
Definition image_cache.h:42
int64_t total_evicted
Definition image_cache.h:38
vector< uint8_t > evicted_mask
Definition image_cache.h:30
int64_t total_reloaded
Definition image_cache.h:39
int64_t current_loaded
Definition image_cache.h:33
int64_t total_loaded
Definition image_cache.h:37
int64_t peak_loaded
Definition image_cache.h:41
void add_tiled_bytes(const size_t bytes)
void clear_range(const size_t begin, const size_t end)
vector< ImageMipLevelStats > mip_levels
std::unique_lock< std::mutex > thread_scoped_lock
Definition thread.h:28
ccl_device_inline bool is_power_of_two(const size_t x)
Definition types_base.h:78
ccl_device_inline size_t divide_up(const size_t x, const size_t y)
Definition types_base.h:54
uint KernelTileDescriptor
ImageDataType
Definition types_image.h:35
@ IMAGE_DATA_NUM_TYPES
Definition types_image.h:51
@ IMAGE_DATA_TYPE_BYTE
Definition types_image.h:40
@ IMAGE_DATA_TYPE_FLOAT
Definition types_image.h:39
@ IMAGE_DATA_TYPE_NANOVDB_FP16
Definition types_image.h:48
@ IMAGE_DATA_TYPE_FLOAT4
Definition types_image.h:36
@ IMAGE_DATA_TYPE_USHORT4
Definition types_image.h:42
@ IMAGE_DATA_TYPE_USHORT
Definition types_image.h:43
@ IMAGE_DATA_TYPE_NANOVDB_EMPTY
Definition types_image.h:49
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT
Definition types_image.h:44
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT3
Definition types_image.h:45
@ IMAGE_DATA_TYPE_HALF
Definition types_image.h:41
@ IMAGE_DATA_TYPE_BYTE4
Definition types_image.h:37
@ IMAGE_DATA_TYPE_HALF4
Definition types_image.h:38
@ IMAGE_DATA_TYPE_NANOVDB_FLOAT4
Definition types_image.h:46
@ IMAGE_DATA_TYPE_NANOVDB_FPN
Definition types_image.h:47
ccl_device_inline bool kernel_tile_descriptor_loaded(const KernelTileDescriptor tile)
InterpolationType
Definition types_image.h:24
@ INTERPOLATION_CLOSEST
Definition types_image.h:27
ccl_device_inline KernelTileDescriptor kernel_tile_descriptor_encode(const uint image_info_id, const uint offset)
ExtensionType
Definition types_image.h:80
@ EXTENSION_EXTEND
Definition types_image.h:84
ccl_device_inline uint kernel_tile_descriptor_offset(const KernelTileDescriptor tile)
ccl_device_inline uint kernel_tile_descriptor_image_info_id(const KernelTileDescriptor tile)