36#ifdef WITH_MEM_VALGRIND
37# include "valgrind/memcheck.h"
45# define POISON_REDZONE_SIZE 32
47# define POISON_REDZONE_SIZE 0
52#define MAKE_ID(a, b, c, d) (int(d) << 24 | int(c) << 16 | (b) << 8 | (a))
53#define MAKE_ID_8(a, b, c, d, e, f, g, h) \
54 (int64_t(h) << 56 | int64_t(g) << 48 | int64_t(f) << 40 | int64_t(e) << 32 | int64_t(d) << 24 | \
55 int64_t(c) << 16 | int64_t(b) << 8 | (a))
64 ((sizeof(void *) > sizeof(int32_t)) ? MAKE_ID_8('e', 'e', 'r', 'f', 'f', 'r', 'e', 'e') : \
65 MAKE_ID('e', 'f', 'f', 'e'))
70#define USEDWORD MAKE_ID('u', 's', 'e', 'd')
131#define MEMPOOL_ELEM_SIZE_MIN (sizeof(void *) * 2)
133#define CHUNK_DATA(chunk) \
134 ((BLI_freenode *)(CHECK_TYPE_INLINE(chunk, BLI_mempool_chunk *), (void *)((chunk) + 1)))
136#define NODE_STEP_NEXT(node) ((BLI_freenode *)((char *)(node) + esize))
137#define NODE_STEP_PREV(node) ((BLI_freenode *)((char *)(node) - esize))
140#define CHUNK_OVERHEAD uint(MEM_SIZE_OVERHEAD + sizeof(BLI_mempool_chunk))
162 while (
index-- && head) {
176 return (elem_num <= pchunk) ? 1 : ((elem_num / pchunk) + 1);
211 mpchunk->
next =
nullptr;
214 if (pool->
free ==
nullptr) [[unlikely]] {
215 pool->
free = curnode;
225#ifdef WITH_MEM_VALGRIND
232#ifdef WITH_MEM_VALGRIND
233 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->
esize);
243#ifdef WITH_MEM_VALGRIND
248#ifdef WITH_MEM_VALGRIND
249 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->
esize);
262#ifdef WITH_MEM_VALGRIND
264 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->
esize);
271#ifdef WITH_MEM_VALGRIND
275 curnode->
next =
nullptr;
277#ifdef WITH_MEM_VALGRIND
278 VALGRIND_MAKE_MEM_UNDEFINED(curnode, pool->
esize);
284#ifdef WITH_MEM_VALGRIND
289#ifdef WITH_MEM_VALGRIND
290 VALGRIND_MAKE_MEM_UNDEFINED(last_tail, pool->
esize);
304#ifdef WITH_MEM_VALGRIND
314 for (; mpchunk; mpchunk = mpchunk_next) {
315 mpchunk_next = mpchunk->
next;
356 pool->
csize = esize * pchunk;
359#if defined(USE_CHUNK_POW2) && !defined(NDEBUG)
368 pool->
free =
nullptr;
374 for (i = 0; i < maxchunks; i++) {
380#ifdef WITH_MEM_VALGRIND
391 if (pool->
free ==
nullptr) [[unlikely]] {
397 free_pop = pool->
free;
400#ifdef WITH_MEM_VALGRIND
421#ifdef WITH_MEM_VALGRIND
425 return static_cast<void *
>(free_pop);
445 for (chunk = pool->
chunks; chunk; chunk = chunk->
next) {
452 BLI_assert_msg(0,
"Attempt to free data which is not in pool.\n");
471 pool->
free = newhead;
477#ifdef WITH_MEM_VALGRIND
478 VALGRIND_MEMPOOL_FREE(pool, addr);
490 first->
next =
nullptr;
494#ifdef WITH_MEM_VALGRIND
499 pool->
free = curnode;
516 curnode->
next =
nullptr;
519#ifdef WITH_MEM_VALGRIND
520 VALGRIND_MEMPOOL_FREE(pool,
CHUNK_DATA(first));
538 if (index < pool->totused) {
560 char *p =
static_cast<char *
>(
data);
567 memcpy(p, elem,
size_t(esize));
575 char *
data =
static_cast<char *
>(
608 for (
size_t i = 1; i < iter_num; i++) {
611 ((*curchunk_threaded_shared) ? (*curchunk_threaded_shared)->next :
nullptr);
622 MEM_delete(iter_arr);
628static void *bli_mempool_iternext(BLI_mempool_iter *iter)
632 if (iter->curchunk ==
nullptr || !iter->pool->totused) {
636 ret = ((
char *)
CHUNK_DATA(iter->curchunk)) + (iter->pool->esize * iter->curindex);
640 if (iter->curindex == iter->pool->pchunk) {
642 iter->curchunk = iter->curchunk->next;
653 ret = bli_mempool_iternext(iter);
663 if (iter->
curchunk ==
nullptr) [[unlikely]] {
674# ifdef WITH_MEM_VALGRIND
684 if (iter->
curchunk ==
nullptr) [[unlikely]] {
686# ifdef WITH_MEM_VALGRIND
693# ifdef WITH_MEM_VALGRIND
710 if (iter->
curchunk ==
nullptr) [[unlikely]] {
723# ifdef WITH_MEM_VALGRIND
743 if (iter->
curchunk ==
nullptr) [[unlikely]] {
746# ifdef WITH_MEM_VALGRIND
758 if (iter->
curchunk ==
nullptr) [[unlikely]] {
761# ifdef WITH_MEM_VALGRIND
776# ifdef WITH_MEM_VALGRIND
800#ifdef WITH_MEM_VALGRIND
805 if (elem_num_reserve == -1) {
814 if (mpchunk && mpchunk->
next) {
816 mpchunk_next = mpchunk->
next;
817 mpchunk->
next =
nullptr;
818 mpchunk = mpchunk_next;
821 mpchunk_next = mpchunk->
next;
823 }
while ((mpchunk = mpchunk_next));
827 pool->
free =
nullptr;
829 chunks_temp = pool->
chunks;
833 while ((mpchunk = chunks_temp)) {
834 chunks_temp = mpchunk->
next;
848#ifdef WITH_MEM_VALGRIND
#define BLI_asan_unpoison(addr, size)
#define BLI_asan_poison(addr, size)
#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)
#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)
#define VALGRIND_DESTROY_MEMPOOL(pool)
#define MEMPOOL_ELEM_SIZE_MIN
#define NODE_STEP_PREV(node)
#define CHUNK_DATA(chunk)
#define POISON_REDZONE_SIZE
#define NODE_STEP_NEXT(node)
Read Blender memory allocation and freeing.
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
ATOMIC_INLINE void * atomic_cas_ptr(void **v, void *old, void *_new)
#define ARRAY_HAS_ITEM(arr_item, arr_start, arr_len)
#define POINTER_OFFSET(v, ofs)
#define BLI_assert_msg(a, msg)
#define MEM_SIZE_OVERHEAD
void * MEM_new_uninitialized(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
void void void BLI_mempool_set_memory_debug()
static BLI_mempool_chunk * mempool_chunk_alloc(const BLI_mempool *pool)
void * MEM_new_array_uninitialized(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
static bool mempool_debug_memset
static void mempool_asan_unlock(BLI_mempool *pool)
static BLI_freenode * mempool_chunk_add(BLI_mempool *pool, BLI_mempool_chunk *mpchunk, BLI_freenode *last_tail)
void BLI_mutex_unlock(ThreadMutex *mutex)
BLI_INLINE uint mempool_maxchunks(const uint elem_num, const uint pchunk)
ParallelMempoolTaskData * mempool_iter_threadsafe_create(BLI_mempool *pool, const size_t iter_num)
MINLINE unsigned int power_of_2_max_u(unsigned int x)
void mempool_iter_threadsafe_destroy(ParallelMempoolTaskData *iter_arr)
void BLI_mempool_free(BLI_mempool *pool, void *addr) ATTR_NONNULL(1
void BLI_mempool_destroy(BLI_mempool *pool) ATTR_NONNULL(1)
void BLI_mutex_lock(ThreadMutex *mutex)
void * MEM_new_zeroed(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
void BLI_mempool_as_array(BLI_mempool *pool, void *data) ATTR_NONNULL(1
BMesh const char void * data
void void * BLI_mempool_as_arrayN(BLI_mempool *pool, const char *allocstr) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1
BLI_mempool * BLI_mempool_create(unsigned int esize, unsigned int elem_num, unsigned int pchunk, unsigned int flag) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL
static void mempool_chunk_free_all(BLI_mempool_chunk *mpchunk, BLI_mempool *pool)
void BLI_mempool_iternew(BLI_mempool *pool, BLI_mempool_iter *iter) ATTR_NONNULL()
void BLI_mempool_clear(BLI_mempool *pool) ATTR_NONNULL(1)
void * BLI_mempool_iterstep(BLI_mempool_iter *iter) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
static void mempool_chunk_free(BLI_mempool_chunk *mpchunk, BLI_mempool *pool)
void void BLI_mempool_clear_ex(BLI_mempool *pool, int elem_num_reserve) ATTR_NONNULL(1)
void * BLI_mempool_calloc(BLI_mempool *pool) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1)
static void mempool_threadsafe_iternew(BLI_mempool *pool, BLI_mempool_threadsafe_iter *ts_iter)
pthread_mutex_t ThreadMutex
void * BLI_mempool_alloc(BLI_mempool *pool) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL ATTR_NONNULL(1)
int BLI_mempool_len(const BLI_mempool *pool) ATTR_NONNULL(1)
static void mempool_asan_lock(BLI_mempool *pool)
void * BLI_mempool_findelem(BLI_mempool *pool, unsigned int index) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
void BLI_mutex_init(ThreadMutex *mutex)
BLI_INLINE BLI_mempool_chunk * mempool_chunk_find(BLI_mempool_chunk *head, uint index)
void * mempool_iter_threadsafe_step(BLI_mempool_threadsafe_iter *ts_iter)
void * MEM_new_array_zeroed(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
struct BLI_mempool_chunk * curchunk
struct BLI_mempool_chunk ** curchunk_threaded_shared
BLI_mempool_chunk * chunk_tail
BLI_mempool_chunk * chunks
BLI_mempool_threadsafe_iter ts_iter