Blender V5.3
BLI_mempool.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2008 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
16
17#include <algorithm>
18#include <cstdlib>
19#include <cstring>
20
21#include "atomic_ops.h"
22
23#include "BLI_utildefines.hh"
24
25#include "BLI_asan.hh"
26#include "BLI_math_base_c.hh"
27#include "BLI_mempool.hh" /* own include */
28#include "BLI_mempool_private.h" /* own include */
29
30#ifdef WITH_ASAN
31# include "BLI_threads.hh"
32#endif
33
34#include "MEM_guardedalloc.h"
35
36#ifdef WITH_MEM_VALGRIND
37# include "valgrind/memcheck.h"
38#endif
39
40#include "BLI_strict_flags.hh" /* IWYU pragma: keep. Keep last. */
41
42namespace blender {
43
44#ifdef WITH_ASAN
45# define POISON_REDZONE_SIZE 32
46#else
47# define POISON_REDZONE_SIZE 0
48#endif
49
50/* NOTE: copied from BLO_core_bhead.hh, don't use here because we're in BLI. */
51/* NOTE: this is endianness-sensitive. */
52#define MAKE_ID(a, b, c, d) (int(d) << 24 | int(c) << 16 | (b) << 8 | (a))
53#define MAKE_ID_8(a, b, c, d, e, f, g, h) \
54 (int64_t(h) << 56 | int64_t(g) << 48 | int64_t(f) << 40 | int64_t(e) << 32 | int64_t(d) << 24 | \
55 int64_t(c) << 16 | int64_t(b) << 8 | (a))
56
63#define FREEWORD \
64 ((sizeof(void *) > sizeof(int32_t)) ? MAKE_ID_8('e', 'e', 'r', 'f', 'f', 'r', 'e', 'e') : \
65 MAKE_ID('e', 'f', 'f', 'e'))
66
70#define USEDWORD MAKE_ID('u', 's', 'e', 'd')
71
72/* optimize pool size */
73#define USE_CHUNK_POW2
74
75#ifndef NDEBUG
76static bool mempool_debug_memset = false;
77#endif
78
90
98
130
131#define MEMPOOL_ELEM_SIZE_MIN (sizeof(void *) * 2)
132
133#define CHUNK_DATA(chunk) \
134 ((BLI_freenode *)(CHECK_TYPE_INLINE(chunk, BLI_mempool_chunk *), (void *)((chunk) + 1)))
135
136#define NODE_STEP_NEXT(node) ((BLI_freenode *)((char *)(node) + esize))
137#define NODE_STEP_PREV(node) ((BLI_freenode *)((char *)(node) - esize))
138
140#define CHUNK_OVERHEAD uint(MEM_SIZE_OVERHEAD + sizeof(BLI_mempool_chunk))
141
143{
144#ifdef WITH_ASAN
145 BLI_mutex_unlock(&pool->mutex);
146#else
147 UNUSED_VARS(pool);
148#endif
149}
150
152{
153#ifdef WITH_ASAN
154 BLI_mutex_lock(&pool->mutex);
155#else
156 UNUSED_VARS(pool);
157#endif
158}
159
161{
162 while (index-- && head) {
163 head = head->next;
164 }
165 return head;
166}
167
174BLI_INLINE uint mempool_maxchunks(const uint elem_num, const uint pchunk)
175{
176 return (elem_num <= pchunk) ? 1 : ((elem_num / pchunk) + 1);
177}
178
180{
181 return static_cast<BLI_mempool_chunk *>(
182 MEM_new_uninitialized(sizeof(BLI_mempool_chunk) + size_t(pool->csize), "mempool chunk"));
183}
184
195 BLI_mempool_chunk *mpchunk,
196 BLI_freenode *last_tail)
197{
198 const uint esize = pool->esize;
199 BLI_freenode *curnode = CHUNK_DATA(mpchunk);
200 uint j;
201
202 /* append */
203 if (pool->chunk_tail) {
204 pool->chunk_tail->next = mpchunk;
205 }
206 else {
207 BLI_assert(pool->chunks == nullptr);
208 pool->chunks = mpchunk;
209 }
210
211 mpchunk->next = nullptr;
212 pool->chunk_tail = mpchunk;
213
214 if (pool->free == nullptr) [[unlikely]] {
215 pool->free = curnode;
216 }
217
218 /* loop through the allocated data, building the pointer structures */
219 j = pool->pchunk;
220 if (pool->flag & BLI_MEMPOOL_ALLOW_ITER) {
221 while (j--) {
222 BLI_freenode *next;
223
225#ifdef WITH_MEM_VALGRIND
226 VALGRIND_MAKE_MEM_DEFINED(curnode, pool->esize - POISON_REDZONE_SIZE);
227#endif
228 curnode->next = next = NODE_STEP_NEXT(curnode);
229 curnode->freeword = FREEWORD;
230
231 BLI_asan_poison(curnode, pool->esize);
232#ifdef WITH_MEM_VALGRIND
233 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->esize);
234#endif
235 curnode = next;
236 }
237 }
238 else {
239 while (j--) {
240 BLI_freenode *next;
241
243#ifdef WITH_MEM_VALGRIND
244 VALGRIND_MAKE_MEM_DEFINED(curnode, pool->esize - POISON_REDZONE_SIZE);
245#endif
246 curnode->next = next = NODE_STEP_NEXT(curnode);
247 BLI_asan_poison(curnode, pool->esize);
248#ifdef WITH_MEM_VALGRIND
249 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->esize);
250#endif
251
252 curnode = next;
253 }
254 }
255
256 /* terminate the list (rewind one)
257 * will be overwritten if 'curnode' gets passed in again as 'last_tail' */
258
259 if (POISON_REDZONE_SIZE > 0) {
261 BLI_asan_poison(curnode, pool->esize);
262#ifdef WITH_MEM_VALGRIND
263 VALGRIND_MAKE_MEM_DEFINED(curnode, pool->esize - POISON_REDZONE_SIZE);
264 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->esize);
265#endif
266 }
267
268 curnode = NODE_STEP_PREV(curnode);
269
271#ifdef WITH_MEM_VALGRIND
272 VALGRIND_MAKE_MEM_DEFINED(curnode, pool->esize - POISON_REDZONE_SIZE);
273#endif
274
275 curnode->next = nullptr;
276 BLI_asan_poison(curnode, pool->esize);
277#ifdef WITH_MEM_VALGRIND
278 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->esize);
279#endif
280
281 /* final pointer in the previously allocated chunk is wrong */
282 if (last_tail) {
283 BLI_asan_unpoison(last_tail, pool->esize - POISON_REDZONE_SIZE);
284#ifdef WITH_MEM_VALGRIND
285 VALGRIND_MAKE_MEM_DEFINED(last_tail, pool->esize - POISON_REDZONE_SIZE);
286#endif
287 last_tail->next = CHUNK_DATA(mpchunk);
288 BLI_asan_poison(last_tail, pool->esize);
289#ifdef WITH_MEM_VALGRIND
290 VALGRIND_MAKE_MEM_UNDEFINED(last_tail, pool->esize);
291#endif
292 }
293
294 return curnode;
295}
296
298{
299#ifdef WITH_ASAN
300 BLI_asan_unpoison(mpchunk, sizeof(BLI_mempool_chunk) + pool->esize * pool->csize);
301#else
302 UNUSED_VARS(pool);
303#endif
304#ifdef WITH_MEM_VALGRIND
305 VALGRIND_MAKE_MEM_DEFINED(mpchunk, sizeof(BLI_mempool_chunk) + pool->esize * pool->csize);
306#endif
307 MEM_delete(mpchunk);
308}
309
311{
312 BLI_mempool_chunk *mpchunk_next;
313
314 for (; mpchunk; mpchunk = mpchunk_next) {
315 mpchunk_next = mpchunk->next;
316 mempool_chunk_free(mpchunk, pool);
317 }
318}
319
320BLI_mempool *BLI_mempool_create(uint esize, uint elem_num, uint pchunk, uint flag)
321{
322 BLI_mempool *pool;
323 BLI_freenode *last_tail = nullptr;
324 uint i, maxchunks;
325
326 /* allocate the pool structure */
327 pool = MEM_new_zeroed<BLI_mempool>("memory pool");
328
329#ifdef WITH_ASAN
330 BLI_mutex_init(&pool->mutex);
331#endif
332
333 /* set the elem size */
334 esize = std::max(esize, uint(MEMPOOL_ELEM_SIZE_MIN));
335
336 if (flag & BLI_MEMPOOL_ALLOW_ITER) {
337 esize = std::max(esize, uint(sizeof(BLI_freenode)));
338 }
339
340 esize += POISON_REDZONE_SIZE;
341
342 maxchunks = mempool_maxchunks(elem_num, pchunk);
343
344 pool->chunks = nullptr;
345 pool->chunk_tail = nullptr;
346 pool->esize = esize;
347
348 /* Optimize chunk size to powers of 2, accounting for slop-space. */
349#ifdef USE_CHUNK_POW2
350 {
351 BLI_assert(power_of_2_max_u(pchunk * esize) > CHUNK_OVERHEAD);
352 pchunk = (power_of_2_max_u(pchunk * esize) - CHUNK_OVERHEAD) / esize;
353 }
354#endif
355
356 pool->csize = esize * pchunk;
357
358 /* Ensure this is a power of 2, minus the rounding by element size. */
359#if defined(USE_CHUNK_POW2) && !defined(NDEBUG)
360 {
361 uint final_size = uint(MEM_SIZE_OVERHEAD) + uint(sizeof(BLI_mempool_chunk)) + pool->csize;
362 BLI_assert((uint(power_of_2_max_u(final_size)) - final_size) < pool->esize);
363 }
364#endif
365
366 pool->pchunk = pchunk;
367 pool->flag = flag;
368 pool->free = nullptr; /* mempool_chunk_add assigns */
369 pool->maxchunks = maxchunks;
370 pool->totused = 0;
371
372 if (elem_num) {
373 /* Allocate the actual chunks. */
374 for (i = 0; i < maxchunks; i++) {
375 BLI_mempool_chunk *mpchunk = mempool_chunk_alloc(pool);
376 last_tail = mempool_chunk_add(pool, mpchunk, last_tail);
377 }
378 }
379
380#ifdef WITH_MEM_VALGRIND
381 VALGRIND_CREATE_MEMPOOL(pool, 0, false);
382#endif
383
384 return pool;
385}
386
388{
389 BLI_freenode *free_pop;
390
391 if (pool->free == nullptr) [[unlikely]] {
392 /* Need to allocate a new chunk. */
393 BLI_mempool_chunk *mpchunk = mempool_chunk_alloc(pool);
394 mempool_chunk_add(pool, mpchunk, nullptr);
395 }
396
397 free_pop = pool->free;
398
400#ifdef WITH_MEM_VALGRIND
401 VALGRIND_MEMPOOL_ALLOC(pool, free_pop, pool->esize - POISON_REDZONE_SIZE);
402 /* Mark as define, then undefine immediately before returning so:
403 * - `free_pop->next` can be read without reading "undefined" memory.
404 * - `freeword` can be set without causing the memory to be considered "defined".
405 *
406 * These could be handled on a more granular level - dealing with defining & underlining these
407 * members explicitly but that requires more involved calls,
408 * adding overhead for no real benefit. */
409 VALGRIND_MAKE_MEM_DEFINED(free_pop, pool->esize - POISON_REDZONE_SIZE);
410#endif
411
412 BLI_assert(pool->chunk_tail->next == nullptr);
413
414 if (pool->flag & BLI_MEMPOOL_ALLOW_ITER) {
415 free_pop->freeword = USEDWORD;
416 }
417
418 pool->free = free_pop->next;
419 pool->totused++;
420
421#ifdef WITH_MEM_VALGRIND
422 VALGRIND_MAKE_MEM_UNDEFINED(free_pop, pool->esize - POISON_REDZONE_SIZE);
423#endif
424
425 return static_cast<void *>(free_pop);
426}
427
429{
430 void *retval = BLI_mempool_alloc(pool);
431
432 memset(retval, 0, size_t(pool->esize) - POISON_REDZONE_SIZE);
433
434 return retval;
435}
436
437void BLI_mempool_free(BLI_mempool *pool, void *addr)
438{
439 BLI_freenode *newhead = static_cast<BLI_freenode *>(addr);
440
441#ifndef NDEBUG
442 {
443 BLI_mempool_chunk *chunk;
444 bool found = false;
445 for (chunk = pool->chunks; chunk; chunk = chunk->next) {
446 if (ARRAY_HAS_ITEM((char *)addr, (char *)CHUNK_DATA(chunk), pool->csize)) {
447 found = true;
448 break;
449 }
450 }
451 if (!found) {
452 BLI_assert_msg(0, "Attempt to free data which is not in pool.\n");
453 }
454 }
455
456 /* Enable for debugging. */
457 if (mempool_debug_memset) [[unlikely]] {
458 memset(addr, 255, pool->esize - POISON_REDZONE_SIZE);
459 }
460#endif
461
462 if (pool->flag & BLI_MEMPOOL_ALLOW_ITER) {
463#ifndef NDEBUG
464 /* This will detect double free's. */
465 BLI_assert(newhead->freeword != FREEWORD);
466#endif
467 newhead->freeword = FREEWORD;
468 }
469
470 newhead->next = pool->free;
471 pool->free = newhead;
472
473 BLI_asan_poison(newhead, pool->esize);
474
475 pool->totused--;
476
477#ifdef WITH_MEM_VALGRIND
478 VALGRIND_MEMPOOL_FREE(pool, addr);
479#endif
480
481 /* Nothing is in use; free all the chunks except the first. */
482 if (UNLIKELY(pool->totused == 0) && (pool->chunks->next)) {
483 const uint esize = pool->esize;
484 BLI_freenode *curnode;
485 uint j;
486 BLI_mempool_chunk *first;
487
488 first = pool->chunks;
489 mempool_chunk_free_all(first->next, pool);
490 first->next = nullptr;
491 pool->chunk_tail = first;
492
493 /* Temporary allocation so VALGRIND doesn't complain when setting freed blocks 'next'. */
494#ifdef WITH_MEM_VALGRIND
495 VALGRIND_MEMPOOL_ALLOC(pool, CHUNK_DATA(first), pool->csize);
496#endif
497
498 curnode = CHUNK_DATA(first);
499 pool->free = curnode;
500
501 j = pool->pchunk;
502 while (j--) {
504 BLI_freenode *next = curnode->next = NODE_STEP_NEXT(curnode);
505 BLI_asan_poison(curnode, pool->esize);
506 curnode = next;
507 }
508
510 BLI_freenode *prev = NODE_STEP_PREV(curnode);
511 BLI_asan_poison(curnode, pool->esize);
512
513 curnode = prev;
514
516 curnode->next = nullptr; /* terminate the list */
517 BLI_asan_poison(curnode, pool->esize);
518
519#ifdef WITH_MEM_VALGRIND
520 VALGRIND_MEMPOOL_FREE(pool, CHUNK_DATA(first));
521#endif
522 }
523}
524
526{
527 int ret = int(pool->totused);
528
529 return ret;
530}
531
533{
534 mempool_asan_lock(pool);
535
537
538 if (index < pool->totused) {
539 /* We could have some faster mem chunk stepping code inline. */
540 BLI_mempool_iter iter;
541 void *elem;
542 BLI_mempool_iternew(pool, &iter);
543 for (elem = BLI_mempool_iterstep(&iter); index-- != 0; elem = BLI_mempool_iterstep(&iter)) {
544 /* pass */
545 }
546
548 return elem;
549 }
550
552 return nullptr;
553}
554
556{
557 const uint esize = pool->esize - uint(POISON_REDZONE_SIZE);
558 BLI_mempool_iter iter;
559 const char *elem;
560 char *p = static_cast<char *>(data);
561
563
564 mempool_asan_lock(pool);
565 BLI_mempool_iternew(pool, &iter);
566 while ((elem = static_cast<const char *>(BLI_mempool_iterstep(&iter)))) {
567 memcpy(p, elem, size_t(esize));
568 p = reinterpret_cast<char *>(NODE_STEP_NEXT(p));
569 }
571}
572
573void *BLI_mempool_as_arrayN(BLI_mempool *pool, const char *allocstr)
574{
575 char *data = static_cast<char *>(
576 MEM_new_array_uninitialized(size_t(pool->totused), pool->esize, allocstr));
578 return data;
579}
580
582{
584
585 iter->pool = pool;
586 iter->curchunk = pool->chunks;
587 iter->curindex = 0;
588}
589
591{
592 BLI_mempool_iternew(pool, &ts_iter->iter);
593 ts_iter->curchunk_threaded_shared = nullptr;
594}
595
597{
599
601 __func__);
602 BLI_mempool_chunk **curchunk_threaded_shared = MEM_new_zeroed<BLI_mempool_chunk *>(__func__);
603
604 mempool_threadsafe_iternew(pool, &iter_arr->ts_iter);
605
606 *curchunk_threaded_shared = iter_arr->ts_iter.iter.curchunk;
607 iter_arr->ts_iter.curchunk_threaded_shared = curchunk_threaded_shared;
608 for (size_t i = 1; i < iter_num; i++) {
609 iter_arr[i].ts_iter = iter_arr[0].ts_iter;
610 *curchunk_threaded_shared = iter_arr[i].ts_iter.iter.curchunk =
611 ((*curchunk_threaded_shared) ? (*curchunk_threaded_shared)->next : nullptr);
612 }
613
614 return iter_arr;
615}
616
618{
619 BLI_assert(iter_arr->ts_iter.curchunk_threaded_shared != nullptr);
620
621 MEM_delete(iter_arr->ts_iter.curchunk_threaded_shared);
622 MEM_delete(iter_arr);
623}
624
625#if 0
626/* unoptimized, more readable */
627
628static void *bli_mempool_iternext(BLI_mempool_iter *iter)
629{
630 void *ret = nullptr;
631
632 if (iter->curchunk == nullptr || !iter->pool->totused) {
633 return ret;
634 }
635
636 ret = ((char *)CHUNK_DATA(iter->curchunk)) + (iter->pool->esize * iter->curindex);
637
638 iter->curindex++;
639
640 if (iter->curindex == iter->pool->pchunk) {
641 iter->curindex = 0;
642 iter->curchunk = iter->curchunk->next;
643 }
644
645 return ret;
646}
647
649{
651
652 do {
653 ret = bli_mempool_iternext(iter);
654 } while (ret && ret->freeword == FREEWORD);
655
656 return ret;
657}
658
659#else /* Optimized version of code above. */
660
662{
663 if (iter->curchunk == nullptr) [[unlikely]] {
664 return nullptr;
665 }
666
667 const uint esize = iter->pool->esize;
668 BLI_freenode *curnode = POINTER_OFFSET(CHUNK_DATA(iter->curchunk), (esize * iter->curindex));
670 do {
671 ret = curnode;
672
674# ifdef WITH_MEM_VALGRIND
675 VALGRIND_MAKE_MEM_DEFINED(ret, iter->pool->esize - POISON_REDZONE_SIZE);
676# endif
677
678 if (++iter->curindex != iter->pool->pchunk) {
679 curnode = POINTER_OFFSET(curnode, esize);
680 }
681 else {
682 iter->curindex = 0;
683 iter->curchunk = iter->curchunk->next;
684 if (iter->curchunk == nullptr) [[unlikely]] {
686# ifdef WITH_MEM_VALGRIND
687 VALGRIND_MAKE_MEM_DEFINED(ret, iter->pool->esize - POISON_REDZONE_SIZE);
688# endif
689 void *ret2 = (ret->freeword == FREEWORD) ? nullptr : ret;
690
691 if (ret->freeword == FREEWORD) {
692 BLI_asan_poison(ret, iter->pool->esize);
693# ifdef WITH_MEM_VALGRIND
694 VALGRIND_MAKE_MEM_UNDEFINED(ret, iter->pool->esize);
695# endif
696 }
697
698 return ret2;
699 }
700 curnode = CHUNK_DATA(iter->curchunk);
701 }
702 } while (ret->freeword == FREEWORD);
703
704 return ret;
705}
706
708{
709 BLI_mempool_iter *iter = &ts_iter->iter;
710 if (iter->curchunk == nullptr) [[unlikely]] {
711 return nullptr;
712 }
713
714 mempool_asan_lock(iter->pool);
715
716 const uint esize = iter->pool->esize;
717 BLI_freenode *curnode = POINTER_OFFSET(CHUNK_DATA(iter->curchunk), (esize * iter->curindex));
719 do {
720 ret = curnode;
721
723# ifdef WITH_MEM_VALGRIND
724 VALGRIND_MAKE_MEM_DEFINED(ret, iter->pool->esize);
725# endif
726
727 if (++iter->curindex != iter->pool->pchunk) {
728 curnode = POINTER_OFFSET(curnode, esize);
729 }
730 else {
731 iter->curindex = 0;
732
733 /* Begin unique to the `threadsafe` version of this function. */
734 for (iter->curchunk = *ts_iter->curchunk_threaded_shared;
735 (iter->curchunk != nullptr) &&
736 (atomic_cas_ptr(reinterpret_cast<void **>(ts_iter->curchunk_threaded_shared),
737 iter->curchunk,
738 iter->curchunk->next) != iter->curchunk);
739 iter->curchunk = *ts_iter->curchunk_threaded_shared)
740 {
741 /* pass. */
742 }
743 if (iter->curchunk == nullptr) [[unlikely]] {
744 if (ret->freeword == FREEWORD) {
745 BLI_asan_poison(ret, esize);
746# ifdef WITH_MEM_VALGRIND
747 VALGRIND_MAKE_MEM_UNDEFINED(ret, iter->pool->esize);
748# endif
750 return nullptr;
751 }
753 return ret;
754 }
755 /* End `threadsafe` exception. */
756
757 iter->curchunk = iter->curchunk->next;
758 if (iter->curchunk == nullptr) [[unlikely]] {
759 if (ret->freeword == FREEWORD) {
760 BLI_asan_poison(ret, iter->pool->esize);
761# ifdef WITH_MEM_VALGRIND
762 VALGRIND_MAKE_MEM_UNDEFINED(ret, iter->pool->esize);
763# endif
765 return nullptr;
766 }
768 return ret;
769 }
770
771 curnode = CHUNK_DATA(iter->curchunk);
772 }
773
774 if (ret->freeword == FREEWORD) {
775 BLI_asan_poison(ret, iter->pool->esize);
776# ifdef WITH_MEM_VALGRIND
777 VALGRIND_MAKE_MEM_UNDEFINED(ret, iter->pool->esize);
778# endif
779 }
780 else {
781 break;
782 }
783 } while (true);
784
786 return ret;
787}
788
789#endif
790
791void BLI_mempool_clear_ex(BLI_mempool *pool, const int elem_num_reserve)
792{
793 BLI_mempool_chunk *mpchunk;
794 BLI_mempool_chunk *mpchunk_next;
795 uint maxchunks;
796
797 BLI_mempool_chunk *chunks_temp;
798 BLI_freenode *last_tail = nullptr;
799
800#ifdef WITH_MEM_VALGRIND
802 VALGRIND_CREATE_MEMPOOL(pool, 0, false);
803#endif
804
805 if (elem_num_reserve == -1) {
806 maxchunks = pool->maxchunks;
807 }
808 else {
809 maxchunks = mempool_maxchunks(uint(elem_num_reserve), pool->pchunk);
810 }
811
812 /* Free all after 'pool->maxchunks'. */
813 mpchunk = mempool_chunk_find(pool->chunks, maxchunks - 1);
814 if (mpchunk && mpchunk->next) {
815 /* terminate */
816 mpchunk_next = mpchunk->next;
817 mpchunk->next = nullptr;
818 mpchunk = mpchunk_next;
819
820 do {
821 mpchunk_next = mpchunk->next;
822 mempool_chunk_free(mpchunk, pool);
823 } while ((mpchunk = mpchunk_next));
824 }
825
826 /* re-initialize */
827 pool->free = nullptr;
828 pool->totused = 0;
829 chunks_temp = pool->chunks;
830 pool->chunks = nullptr;
831 pool->chunk_tail = nullptr;
832
833 while ((mpchunk = chunks_temp)) {
834 chunks_temp = mpchunk->next;
835 last_tail = mempool_chunk_add(pool, mpchunk, last_tail);
836 }
837}
838
840{
841 BLI_mempool_clear_ex(pool, -1);
842}
843
845{
846 mempool_chunk_free_all(pool->chunks, pool);
847
848#ifdef WITH_MEM_VALGRIND
850#endif
851
852 MEM_delete(pool);
853}
854
855#ifndef NDEBUG
860#endif
861
862} // namespace blender
#define BLI_asan_unpoison(addr, size)
Definition BLI_asan.hh:36
#define BLI_asan_poison(addr, size)
Definition BLI_asan.hh:31
#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)
#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)
#define VALGRIND_DESTROY_MEMPOOL(pool)
#define MEMPOOL_ELEM_SIZE_MIN
#define NODE_STEP_PREV(node)
#define USEDWORD
#define CHUNK_DATA(chunk)
#define FREEWORD
#define POISON_REDZONE_SIZE
#define CHUNK_OVERHEAD
#define NODE_STEP_NEXT(node)
Read Blender memory allocation and freeing.
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
ATOMIC_INLINE void * atomic_cas_ptr(void **v, void *old, void *_new)
#define UNUSED_VARS(...)
#define ARRAY_HAS_ITEM(arr_item, arr_start, arr_len)
#define BLI_INLINE
#define UNLIKELY(x)
#define POINTER_OFFSET(v, ofs)
#define BLI_assert_msg(a, msg)
#define MEM_SIZE_OVERHEAD
BMesh Iterators.
void * MEM_new_uninitialized(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
void void void BLI_mempool_set_memory_debug()
static BLI_mempool_chunk * mempool_chunk_alloc(const BLI_mempool *pool)
unsigned int uint
Definition BKE_text.h:55
void * MEM_new_array_uninitialized(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
static bool mempool_debug_memset
static void mempool_asan_unlock(BLI_mempool *pool)
static BLI_freenode * mempool_chunk_add(BLI_mempool *pool, BLI_mempool_chunk *mpchunk, BLI_freenode *last_tail)
void BLI_mutex_unlock(ThreadMutex *mutex)
Definition threads.cc:353
BLI_INLINE uint mempool_maxchunks(const uint elem_num, const uint pchunk)
ParallelMempoolTaskData * mempool_iter_threadsafe_create(BLI_mempool *pool, const size_t iter_num)
MINLINE unsigned int power_of_2_max_u(unsigned int x)
void mempool_iter_threadsafe_destroy(ParallelMempoolTaskData *iter_arr)
void BLI_mempool_free(BLI_mempool *pool, void *addr) ATTR_NONNULL(1
void BLI_mempool_destroy(BLI_mempool *pool) ATTR_NONNULL(1)
void BLI_mutex_lock(ThreadMutex *mutex)
Definition threads.cc:348
void * MEM_new_zeroed(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
void BLI_mempool_as_array(BLI_mempool *pool, void *data) ATTR_NONNULL(1
BMesh const char void * data
void void * BLI_mempool_as_arrayN(BLI_mempool *pool, const char *allocstr) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
BLI_mempool * BLI_mempool_create(unsigned int esize, unsigned int elem_num, unsigned int pchunk, unsigned int flag) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL
static void mempool_chunk_free_all(BLI_mempool_chunk *mpchunk, BLI_mempool *pool)
void BLI_mempool_iternew(BLI_mempool *pool, BLI_mempool_iter *iter) ATTR_NONNULL()
void BLI_mempool_clear(BLI_mempool *pool) ATTR_NONNULL(1)
void * BLI_mempool_iterstep(BLI_mempool_iter *iter) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
static void mempool_chunk_free(BLI_mempool_chunk *mpchunk, BLI_mempool *pool)
void void BLI_mempool_clear_ex(BLI_mempool *pool, int elem_num_reserve) ATTR_NONNULL(1)
@ BLI_MEMPOOL_ALLOW_ITER
void * BLI_mempool_calloc(BLI_mempool *pool) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1)
static void mempool_threadsafe_iternew(BLI_mempool *pool, BLI_mempool_threadsafe_iter *ts_iter)
pthread_mutex_t ThreadMutex
void * BLI_mempool_alloc(BLI_mempool *pool) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1)
int BLI_mempool_len(const BLI_mempool *pool) ATTR_NONNULL(1)
static void mempool_asan_lock(BLI_mempool *pool)
void * BLI_mempool_findelem(BLI_mempool *pool, unsigned int index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
void BLI_mutex_init(ThreadMutex *mutex)
Definition threads.cc:343
BLI_INLINE BLI_mempool_chunk * mempool_chunk_find(BLI_mempool_chunk *head, uint index)
void * mempool_iter_threadsafe_step(BLI_mempool_threadsafe_iter *ts_iter)
void * MEM_new_array_zeroed(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
_W64 int intptr_t
Definition BKE_text.h:119
BLI_freenode * next
BLI_mempool_chunk * next
struct BLI_mempool_chunk * curchunk
struct BLI_mempool_chunk ** curchunk_threaded_shared
BLI_freenode * free
BLI_mempool_chunk * chunk_tail
BLI_mempool_chunk * chunks
BLI_mempool_threadsafe_iter ts_iter