Blender V5.3
mallocn_lockfree_impl.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2013-2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
10
11#include <stdarg.h>
12#include <stdio.h> /* printf */
13#include <stdlib.h>
14#include <string.h> /* memcpy */
15#include <sys/types.h>
16
17#include "PRF_profile.hh"
18
19#include "MEM_guardedalloc.h"
20#include "MEM_safe_multiply.h"
21
22/* Quiet warnings when dealing with allocated data written into the blend file.
23 * This also rounds up and causes warnings which we don't consider bugs in practice. */
24#ifdef WITH_MEM_VALGRIND
25# include "valgrind/memcheck.h"
26#endif
27
28/* to ensure strict conversions */
29#include "../../source/blender/blenlib/BLI_strict_flags.hh"
30
31#include "atomic_ops.h"
32#include "mallocn_intern.hh"
34
35using namespace mem_guarded::internal;
36
37namespace {
38
39typedef struct MemHead {
40 /* Length of allocated memory block. */
41 size_t len;
42} MemHead;
43static_assert(MEM_MIN_CPP_ALIGNMENT <= alignof(MemHead), "Bad alignment of MemHead");
44static_assert(MEM_MIN_CPP_ALIGNMENT <= sizeof(MemHead), "Bad size of MemHead");
45
46typedef struct MemHeadAligned {
47 short alignment;
48 size_t len;
49} MemHeadAligned;
50static_assert(MEM_MIN_CPP_ALIGNMENT <= alignof(MemHeadAligned), "Bad alignment of MemHeadAligned");
51static_assert(MEM_MIN_CPP_ALIGNMENT <= sizeof(MemHeadAligned), "Bad size of MemHeadAligned");
52
53} // namespace
54
55static bool malloc_debug_memset = false;
56
57static void (*error_callback)(const char *) = nullptr;
58
63enum {
71
72 MEMHEAD_FLAG_MASK = (1 << 2) - 1
73};
74
75#define MEMHEAD_FROM_PTR(ptr) (((MemHead *)ptr) - 1)
76#define PTR_FROM_MEMHEAD(memhead) (memhead + 1)
77#define MEMHEAD_ALIGNED_FROM_PTR(ptr) (((MemHeadAligned *)ptr) - 1)
78#define MEMHEAD_IS_ALIGNED(memhead) ((memhead)->len & size_t(MEMHEAD_FLAG_ALIGN))
79#define MEMHEAD_HAS_NONTRIVIAL_DESTRUCTOR(memhead) \
80 ((memhead)->len & size_t(MEMHEAD_FLAG_NONTRIVIAL_DESTRUCTOR))
81#define MEMHEAD_LEN(memhead) ((memhead)->len & ~size_t(MEMHEAD_FLAG_MASK))
82
83#ifdef __GNUC__
84__attribute__((format(printf, 1, 0)))
85#endif
86static void print_error(const char *message, va_list str_format_args)
87{
88 char buf[512];
89 vsnprintf(buf, sizeof(buf), message, str_format_args);
90 buf[sizeof(buf) - 1] = '\0';
91
92 if (error_callback) {
93 error_callback(buf);
94 }
95}
96
97#ifdef __GNUC__
99#endif
100static void print_error(const char *message, ...)
101{
102 va_list str_format_args;
103 va_start(str_format_args, message);
104 print_error(message, str_format_args);
105 va_end(str_format_args);
106}
107
108#ifdef __GNUC__
110#endif
111static void report_error_on_address(const void *vmemh, const char *message, ...)
112{
113 va_list str_format_args;
114
115 va_start(str_format_args, message);
116 print_error(message, str_format_args);
117 va_end(str_format_args);
118
119 if (vmemh == nullptr) {
121 return;
122 }
123
124 const MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
125 const size_t len = MEMHEAD_LEN(memh);
126
127 const void *address = memh;
128 size_t size = len + sizeof(*memh);
129 if (MEMHEAD_IS_ALIGNED(memh)) [[unlikely]] {
130 const MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
131 address = MEMHEAD_REAL_PTR(memh_aligned);
132 size = len + sizeof(*memh_aligned) + MEMHEAD_ALIGN_PADDING(memh_aligned->alignment);
133 }
135}
136
137size_t MEM_lockfree_allocN_len(const void *vmemh)
138{
139 if (vmemh) [[likely]] {
140 return MEMHEAD_LEN(MEMHEAD_FROM_PTR(vmemh));
141 }
142
143 return 0;
144}
145
146void MEM_lockfree_freeN(void *vmemh, DestructorType destructor_type)
147{
149 if (leak_detector_has_run) [[unlikely]] {
151 }
152
153 if (vmemh == nullptr) [[unlikely]] {
154 report_error_on_address(vmemh, "Attempt to free nullptr pointer\n");
155 return;
156 }
157
158 MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
159 size_t len = MEMHEAD_LEN(memh);
160
161 if (destructor_type != DestructorType::NonTrivial && MEMHEAD_HAS_NONTRIVIAL_DESTRUCTOR(memh)) {
163 "Attempt to use C-style MEM_delete_void on a pointer created with "
164 "CPP-style MEM_new or new\n");
165 }
166
168
169 if (malloc_debug_memset && len) [[unlikely]] {
170 memset(memh + 1, 255, len);
171 }
172 if (MEMHEAD_IS_ALIGNED(memh)) [[unlikely]] {
173 MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
174 aligned_free(MEMHEAD_REAL_PTR(memh_aligned));
175 }
176 else {
177 PRF_memory_free(memh);
178 free(memh);
179 }
180}
181
182void *MEM_lockfree_dupallocN(const void *vmemh)
183{
185 void *newp = nullptr;
186 if (vmemh) {
187 const MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
188 const size_t prev_size = MEM_lockfree_allocN_len(vmemh);
189
192 "Attempt to use C-style MEM_dupalloc_void on a pointer created with "
193 "CPP-style MEM_new or new\n");
194 }
195
196 if (MEMHEAD_IS_ALIGNED(memh)) [[unlikely]] {
197 const MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
199 prev_size, size_t(memh_aligned->alignment), "dupli_malloc", DestructorType::Trivial);
200 }
201 else {
202 newp = MEM_lockfree_mallocN(prev_size, "dupli_malloc");
203 }
204 memcpy(newp, vmemh, prev_size);
205 }
206 return newp;
207}
208
209void *MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *str)
210{
212 void *newp = nullptr;
213
214 if (vmemh) {
215 const MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
216 const size_t old_len = MEM_lockfree_allocN_len(vmemh);
217
220 vmemh,
221 "Attempt to use C-style MEM_realloc_uninitialized on a pointer created with "
222 "CPP-style MEM_new or new\n");
223 }
224
225 if (!MEMHEAD_IS_ALIGNED(memh)) [[likely]] {
226 newp = MEM_lockfree_mallocN(len, "realloc");
227 }
228 else {
229 const MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
231 len, size_t(memh_aligned->alignment), "realloc", DestructorType::Trivial);
232 }
233
234 if (newp) {
235 if (len < old_len) {
236 /* shrink */
237 memcpy(newp, vmemh, len);
238 }
239 else {
240 /* grow (or remain same size) */
241 memcpy(newp, vmemh, old_len);
242 }
243 }
244
246 }
247 else {
249 }
250
251 return newp;
252}
253
254void *MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *str)
255{
257 void *newp = nullptr;
258
259 if (vmemh) {
260 const MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
261 const size_t old_len = MEM_lockfree_allocN_len(vmemh);
262
265 vmemh,
266 "Attempt to use C-style MEM_realloc_zeroed on a pointer created with "
267 "CPP-style MEM_new or new\n");
268 }
269
270 if (!MEMHEAD_IS_ALIGNED(memh)) [[likely]] {
271 newp = MEM_lockfree_mallocN(len, "recalloc");
272 }
273 else {
274 const MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
276 len, size_t(memh_aligned->alignment), "recalloc", DestructorType::Trivial);
277 }
278
279 if (newp) {
280 if (len < old_len) {
281 /* shrink */
282 memcpy(newp, vmemh, len);
283 }
284 else {
285 memcpy(newp, vmemh, old_len);
286
287 if (len > old_len) {
288 /* grow */
289 /* zero new bytes */
290 memset(((char *)newp) + old_len, 0, len - old_len);
291 }
292 }
293 }
294
296 }
297 else {
299 }
300
301 return newp;
302}
303
304void *MEM_lockfree_callocN(size_t len, const char *str)
305{
307 MemHead *memh;
308
310
311 memh = (MemHead *)calloc(1, len + sizeof(MemHead));
312 PRF_memory_alloc(memh, len + sizeof(MemHead));
313
314 if (memh) [[likely]] {
315 memh->len = len;
317
318 return PTR_FROM_MEMHEAD(memh);
319 }
320 print_error("Calloc returns null: len=" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
321 SIZET_ARG(len),
322 str,
324 return nullptr;
325}
326
327void *MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *str)
328{
330 size_t total_size;
331 if (!MEM_size_safe_multiply(len, size, &total_size)) [[unlikely]] {
333 "Calloc array aborted due to integer overflow: "
334 "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
335 SIZET_ARG(len),
336 SIZET_ARG(size),
337 str,
339 abort();
340 return nullptr;
341 }
342
343 return MEM_lockfree_callocN(total_size, str);
344}
345
346void *MEM_lockfree_mallocN(size_t len, const char *str)
347{
349 MemHead *memh;
350
351#ifdef WITH_MEM_VALGRIND
352 const size_t len_unaligned = len;
353#endif
355
356 memh = (MemHead *)malloc(len + sizeof(MemHead));
357 PRF_memory_alloc(memh, len + sizeof(MemHead));
358
359 if (memh) [[likely]] {
360
361 if (len) [[likely]] {
362 if (malloc_debug_memset) [[unlikely]] {
363 memset(memh + 1, 255, len);
364 }
365#ifdef WITH_MEM_VALGRIND
367 VALGRIND_MAKE_MEM_UNDEFINED(memh + 1, len_unaligned);
368 }
369 else {
370 VALGRIND_MAKE_MEM_DEFINED((const char *)(memh + 1) + len_unaligned, len - len_unaligned);
371 }
372#endif /* WITH_MEM_VALGRIND */
373 }
374
375 memh->len = len;
377
378 return PTR_FROM_MEMHEAD(memh);
379 }
380 print_error("Malloc returns null: len=" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
381 SIZET_ARG(len),
382 str,
384 return nullptr;
385}
386
387void *MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *str)
388{
390 size_t total_size;
391 if (!MEM_size_safe_multiply(len, size, &total_size)) [[unlikely]] {
393 "Malloc array aborted due to integer overflow: "
394 "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
395 SIZET_ARG(len),
396 SIZET_ARG(size),
397 str,
399 abort();
400 return nullptr;
401 }
402
403 return MEM_lockfree_mallocN(total_size, str);
404}
405
407 size_t alignment,
408 const char *str,
409 const DestructorType destructor_type)
410{
412 /* Huge alignment values doesn't make sense and they wouldn't fit into 'short' used in the
413 * MemHead. */
414 assert(alignment < 1024);
415
416 /* We only support alignments that are a power of two. */
417 assert(IS_POW2(alignment));
418
419 /* Some OS specific aligned allocators require a certain minimal alignment. */
420 if (alignment < ALIGNED_MALLOC_MINIMUM_ALIGNMENT) {
422 }
423
424 /* It's possible that MemHead's size is not properly aligned,
425 * do extra padding to deal with this.
426 *
427 * We only support small alignments which fits into short in
428 * order to save some bits in MemHead structure.
429 */
430 size_t extra_padding = MEMHEAD_ALIGN_PADDING(alignment);
431
432#ifdef WITH_MEM_VALGRIND
433 const size_t len_unaligned = len;
434#endif
436
437 MemHeadAligned *memh = (MemHeadAligned *)aligned_malloc(
438 len + extra_padding + sizeof(MemHeadAligned), alignment);
439 PRF_memory_alloc(memh, len + extra_padding + sizeof(MemHeadAligned));
440
441 if (memh) [[likely]] {
442 /* We keep padding in the beginning of MemHead,
443 * this way it's always possible to get MemHead
444 * from the data pointer.
445 */
446 memh = (MemHeadAligned *)((char *)memh + extra_padding);
447
448 if (len) [[likely]] {
449 if (malloc_debug_memset) [[unlikely]] {
450 memset(memh + 1, 255, len);
451 }
452#ifdef WITH_MEM_VALGRIND
454 VALGRIND_MAKE_MEM_UNDEFINED(memh + 1, len_unaligned);
455 }
456 else {
457 VALGRIND_MAKE_MEM_DEFINED((const char *)(memh + 1) + len_unaligned, len - len_unaligned);
458 }
459#endif /* WITH_MEM_VALGRIND */
460 }
461
462 memh->len = len | size_t(MEMHEAD_FLAG_ALIGN) |
463 size_t(destructor_type == DestructorType::NonTrivial ?
465 0);
466 memh->alignment = short(alignment);
468
469 return PTR_FROM_MEMHEAD(memh);
470 }
471 print_error("Malloc returns null: len=" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
472 SIZET_ARG(len),
473 str,
475 return nullptr;
476}
477
478static void *mem_lockfree_malloc_arrayN_aligned(const size_t len,
479 const size_t size,
480 const size_t alignment,
481 const char *str,
482 size_t &r_bytes_num)
483{
484 if (!MEM_size_safe_multiply(len, size, &r_bytes_num)) [[unlikely]] {
486 "Calloc array aborted due to integer overflow: "
487 "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total " SIZET_FORMAT "\n",
488 SIZET_ARG(len),
489 SIZET_ARG(size),
490 str,
492 abort();
493 return nullptr;
494 }
495 if (alignment <= MEM_MIN_CPP_ALIGNMENT) {
496 return mem_mallocN(r_bytes_num, str);
497 }
498 void *ptr = MEM_new_uninitialized_aligned(r_bytes_num, alignment, str);
499 return ptr;
500}
501
503 const size_t size,
504 const size_t alignment,
505 const char *str)
506{
508 size_t bytes_num;
509 return mem_lockfree_malloc_arrayN_aligned(len, size, alignment, str, bytes_num);
510}
511
513 const size_t size,
514 const size_t alignment,
515 const char *str)
516{
518 /* There is no lower level #calloc with an alignment parameter, so unless the alignment is less
519 * than or equal to what we'd get by default, we have to fall back to #memset unfortunately. */
520 if (alignment <= MEM_MIN_CPP_ALIGNMENT) {
521 return MEM_lockfree_calloc_arrayN(len, size, str);
522 }
523
524 size_t bytes_num;
525 void *ptr = mem_lockfree_malloc_arrayN_aligned(len, size, alignment, str, bytes_num);
526 if (!ptr) {
527 return nullptr;
528 }
529 memset(ptr, 0, bytes_num);
530 return ptr;
531}
532
534
536
538
539/* Unused. */
540
541void MEM_lockfree_callbackmemlist(void (*func)(void *))
542{
543 (void)func; /* Ignored. */
544}
545
547{
548 printf("\ntotal memory len: %.3f MB\n", double(memory_usage_current()) / double(1024 * 1024));
549 printf("peak memory len: %.3f MB\n", double(memory_usage_peak()) / double(1024 * 1024));
550 printf(
551 "\nFor more detailed per-block statistics run Blender with memory debugging command line "
552 "argument.\n");
553
554#ifdef HAVE_MALLOC_STATS
555 printf("System Statistics:\n");
556 malloc_stats();
557#endif
558}
559
560void MEM_lockfree_set_error_callback(void (*func)(const char *))
561{
562 error_callback = func;
563}
564
566{
567 return true;
568}
569
574
579
584
585/* Dummy. */
586
591
593{
594 return memory_usage_peak();
595}
596
597#ifndef NDEBUG
598const char *MEM_lockfree_name_ptr(void *vmemh)
599{
600 if (vmemh) {
601 return "unknown block name ptr";
602 }
603
604 return "MEM_lockfree_name_ptr(nullptr)";
605}
606
607void MEM_lockfree_name_ptr_set(void * /*vmemh*/, const char * /*str*/) {}
608#endif /* !NDEBUG */
Read Blender memory allocation and freeing.
bool MEM_size_safe_multiply(size_t a, size_t b, size_t *result)
#define PRF_memory_free(ptr)
#define PRF_scope(category)
#define PRF_memory_alloc(ptr, size)
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
struct AtomicSpinLock __attribute__((aligned(32))) AtomicSpinLock
#define printf
#define str(s)
#define assert(assertion)
bool leak_detector_has_run
char free_after_leak_detection_message[]
void aligned_free(void *ptr)
Definition mallocn.cc:99
void * MEM_new_uninitialized_aligned(size_t len, size_t alignment, const char *str)
Definition mallocn.cc:134
void * aligned_malloc(size_t size, size_t alignment)
Definition mallocn.cc:77
@ MEMHEAD_FLAG_NONTRIVIAL_DESTRUCTOR
MEM_INLINE void MEM_trigger_error_on_memory_block(const void *, const size_t)
size_t memory_usage_current(void)
void * MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
#define MEMHEAD_REAL_PTR(memh)
#define IS_POW2(a)
#define MEMHEAD_ALIGN_PADDING(alignment)
void * MEM_lockfree_dupallocN(const void *vmemh) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT
void memory_usage_block_alloc(size_t size)
void MEM_lockfree_reset_peak_memory(void)
void mem_lockfree_clearmemlist(void)
#define SIZET_ARG(a)
void MEM_lockfree_name_ptr_set(void *vmemh, const char *str)
void * MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
void MEM_lockfree_set_error_callback(void(*func)(const char *))
void MEM_lockfree_printmemlist_stats(void)
#define SIZET_ALIGN_4(len)
void MEM_lockfree_freeN(void *vmemh, mem_guarded::internal::DestructorType destructor_type)
size_t memory_usage_block_num(void)
void MEM_lockfree_set_memory_debug(void)
void * MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2)
bool MEM_lockfree_consistency_check(void)
void * MEM_lockfree_callocN(size_t len, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
#define SIZET_FORMAT
void * MEM_lockfree_mallocN_aligned(size_t len, size_t alignment, const char *str, mem_guarded::internal::DestructorType destructor_type) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(3)
void memory_usage_block_free(size_t size)
size_t memory_usage_peak(void)
void MEM_lockfree_printmemlist_pydict(void)
#define ALIGNED_MALLOC_MINIMUM_ALIGNMENT
void * MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2)
void memory_usage_peak_reset(void)
void * MEM_lockfree_malloc_arrayN_aligned(size_t len, size_t size, size_t alignment, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
void * MEM_lockfree_mallocN(size_t len, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
void MEM_lockfree_printmemlist(void)
size_t MEM_lockfree_allocN_len(const void *vmemh) ATTR_WARN_UNUSED_RESULT
const char * MEM_lockfree_name_ptr(void *vmemh)
void MEM_lockfree_callbackmemlist(void(*func)(void *))
void * MEM_lockfree_calloc_arrayN_aligned(size_t len, size_t size, size_t alignment, const char *str) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
size_t MEM_lockfree_get_memory_in_use(void)
unsigned int MEM_lockfree_get_memory_blocks_in_use(void)
size_t MEM_lockfree_get_peak_memory(void) ATTR_WARN_UNUSED_RESULT
static void * mem_lockfree_malloc_arrayN_aligned(const size_t len, const size_t size, const size_t alignment, const char *str, size_t &r_bytes_num)
static void(*) error_callback(const char *)
#define PTR_FROM_MEMHEAD(memhead)
static bool malloc_debug_memset
@ MEMHEAD_FLAG_MASK
@ MEMHEAD_FLAG_ALIGN
#define MEMHEAD_HAS_NONTRIVIAL_DESTRUCTOR(memhead)
static void print_error(const char *message, va_list str_format_args)
static void report_error_on_address(const void *vmemh, const char *message,...)
#define MEMHEAD_LEN(memhead)
#define MEMHEAD_IS_ALIGNED(memhead)
#define MEMHEAD_FROM_PTR(ptr)
#define MEMHEAD_ALIGNED_FROM_PTR(ptr)
ccl_device_inline dual1 len(const ccl_private dual< T > &a)
Definition math_dual.h:263
std::string format(const std::string &fmt, Args &&...args)
Definition diagnostic.hh:43
void *(*) mem_mallocN(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
Definition mallocn.cc:41