1 #define JEMALLOC_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /******************************************************************************/
5 /* Data. */
6
7 malloc_tsd_data(, arenas, arena_t *, NULL)
8 malloc_tsd_data(, thread_allocated, thread_allocated_t,
9 THREAD_ALLOCATED_INITIALIZER)
10
11 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
12 const char *__malloc_options_1_0 = NULL;
13 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
14
15 /* Runtime configuration options. */
16 const char *je_malloc_conf;
17 bool opt_abort =
18 #ifdef JEMALLOC_DEBUG
19 true
20 #else
21 false
22 #endif
23 ;
24 bool opt_junk =
25 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
26 true
27 #else
28 false
29 #endif
30 ;
31 size_t opt_quarantine = ZU(0);
32 bool opt_redzone = false;
33 bool opt_utrace = false;
34 bool opt_valgrind = false;
35 bool opt_xmalloc = false;
36 bool opt_zero = false;
37 size_t opt_narenas = 0;
38
39 unsigned ncpus;
40
41 malloc_mutex_t arenas_lock;
42 arena_t **arenas;
43 unsigned narenas_total;
44 unsigned narenas_auto;
45
46 /* Set to true once the allocator has been initialized. */
47 static bool malloc_initialized = false;
48
49 #ifdef JEMALLOC_THREADED_INIT
50 /* Used to let the initializing thread recursively allocate. */
51 # define NO_INITIALIZER ((unsigned long)0)
52 # define INITIALIZER pthread_self()
53 # define IS_INITIALIZER (malloc_initializer == pthread_self())
54 static pthread_t malloc_initializer = NO_INITIALIZER;
55 #else
56 # define NO_INITIALIZER false
57 # define INITIALIZER true
58 # define IS_INITIALIZER malloc_initializer
59 static bool malloc_initializer = NO_INITIALIZER;
60 #endif
61
62 /* Used to avoid initialization races. */
63 #ifdef _WIN32
64 static malloc_mutex_t init_lock;
65
JEMALLOC_ATTR(constructor)66 JEMALLOC_ATTR(constructor)
67 static void WINAPI
68 _init_init_lock(void)
69 {
70
71 malloc_mutex_init(&init_lock);
72 }
73
74 #ifdef _MSC_VER
75 # pragma section(".CRT$XCU", read)
76 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
77 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
78 #endif
79
80 #else
81 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
82 #endif
83
84 typedef struct {
85 void *p; /* Input pointer (as in realloc(p, s)). */
86 size_t s; /* Request size. */
87 void *r; /* Result pointer. */
88 } malloc_utrace_t;
89
90 #ifdef JEMALLOC_UTRACE
91 # define UTRACE(a, b, c) do { \
92 if (opt_utrace) { \
93 int utrace_serrno = errno; \
94 malloc_utrace_t ut; \
95 ut.p = (a); \
96 ut.s = (b); \
97 ut.r = (c); \
98 utrace(&ut, sizeof(ut)); \
99 errno = utrace_serrno; \
100 } \
101 } while (0)
102 #else
103 # define UTRACE(a, b, c)
104 #endif
105
106 /******************************************************************************/
107 /*
108 * Function prototypes for static functions that are referenced prior to
109 * definition.
110 */
111
112 static bool malloc_init_hard(void);
113
114 /******************************************************************************/
115 /*
116 * Begin miscellaneous support functions.
117 */
118
119 /* Create a new arena and insert it into the arenas array at index ind. */
120 arena_t *
arenas_extend(unsigned ind)121 arenas_extend(unsigned ind)
122 {
123 arena_t *ret;
124
125 ret = (arena_t *)base_alloc(sizeof(arena_t));
126 if (ret != NULL && arena_new(ret, ind) == false) {
127 arenas[ind] = ret;
128 return (ret);
129 }
130 /* Only reached if there is an OOM error. */
131
132 /*
133 * OOM here is quite inconvenient to propagate, since dealing with it
134 * would require a check for failure in the fast path. Instead, punt
135 * by using arenas[0]. In practice, this is an extremely unlikely
136 * failure.
137 */
138 malloc_write("<jemalloc>: Error initializing arena\n");
139 if (opt_abort)
140 abort();
141
142 return (arenas[0]);
143 }
144
145 /* Slow path, called only by choose_arena(). */
146 arena_t *
choose_arena_hard(void)147 choose_arena_hard(void)
148 {
149 arena_t *ret;
150
151 if (narenas_auto > 1) {
152 unsigned i, choose, first_null;
153
154 choose = 0;
155 first_null = narenas_auto;
156 malloc_mutex_lock(&arenas_lock);
157 assert(arenas[0] != NULL);
158 for (i = 1; i < narenas_auto; i++) {
159 if (arenas[i] != NULL) {
160 /*
161 * Choose the first arena that has the lowest
162 * number of threads assigned to it.
163 */
164 if (arenas[i]->nthreads <
165 arenas[choose]->nthreads)
166 choose = i;
167 } else if (first_null == narenas_auto) {
168 /*
169 * Record the index of the first uninitialized
170 * arena, in case all extant arenas are in use.
171 *
172 * NB: It is possible for there to be
173 * discontinuities in terms of initialized
174 * versus uninitialized arenas, due to the
175 * "thread.arena" mallctl.
176 */
177 first_null = i;
178 }
179 }
180
181 if (arenas[choose]->nthreads == 0
182 || first_null == narenas_auto) {
183 /*
184 * Use an unloaded arena, or the least loaded arena if
185 * all arenas are already initialized.
186 */
187 ret = arenas[choose];
188 } else {
189 /* Initialize a new arena. */
190 ret = arenas_extend(first_null);
191 }
192 ret->nthreads++;
193 malloc_mutex_unlock(&arenas_lock);
194 } else {
195 ret = arenas[0];
196 malloc_mutex_lock(&arenas_lock);
197 ret->nthreads++;
198 malloc_mutex_unlock(&arenas_lock);
199 }
200
201 arenas_tsd_set(&ret);
202
203 return (ret);
204 }
205
206 static void
stats_print_atexit(void)207 stats_print_atexit(void)
208 {
209
210 if (config_tcache && config_stats) {
211 unsigned narenas, i;
212
213 /*
214 * Merge stats from extant threads. This is racy, since
215 * individual threads do not lock when recording tcache stats
216 * events. As a consequence, the final stats may be slightly
217 * out of date by the time they are reported, if other threads
218 * continue to allocate.
219 */
220 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
221 arena_t *arena = arenas[i];
222 if (arena != NULL) {
223 tcache_t *tcache;
224
225 /*
226 * tcache_stats_merge() locks bins, so if any
227 * code is introduced that acquires both arena
228 * and bin locks in the opposite order,
229 * deadlocks may result.
230 */
231 malloc_mutex_lock(&arena->lock);
232 ql_foreach(tcache, &arena->tcache_ql, link) {
233 tcache_stats_merge(tcache, arena);
234 }
235 malloc_mutex_unlock(&arena->lock);
236 }
237 }
238 }
239 je_malloc_stats_print(NULL, NULL, NULL);
240 }
241
242 /*
243 * End miscellaneous support functions.
244 */
245 /******************************************************************************/
246 /*
247 * Begin initialization functions.
248 */
249
250 static unsigned
malloc_ncpus(void)251 malloc_ncpus(void)
252 {
253 long result;
254
255 #ifdef _WIN32
256 SYSTEM_INFO si;
257 GetSystemInfo(&si);
258 result = si.dwNumberOfProcessors;
259 #else
260 result = sysconf(_SC_NPROCESSORS_ONLN);
261 #endif
262 return ((result == -1) ? 1 : (unsigned)result);
263 }
264
265 void
arenas_cleanup(void * arg)266 arenas_cleanup(void *arg)
267 {
268 arena_t *arena = *(arena_t **)arg;
269
270 malloc_mutex_lock(&arenas_lock);
271 arena->nthreads--;
272 malloc_mutex_unlock(&arenas_lock);
273 }
274
275 JEMALLOC_ALWAYS_INLINE_C void
malloc_thread_init(void)276 malloc_thread_init(void)
277 {
278
279 /*
280 * TSD initialization can't be safely done as a side effect of
281 * deallocation, because it is possible for a thread to do nothing but
282 * deallocate its TLS data via free(), in which case writing to TLS
283 * would cause write-after-free memory corruption. The quarantine
284 * facility *only* gets used as a side effect of deallocation, so make
285 * a best effort attempt at initializing its TSD by hooking all
286 * allocation events.
287 */
288 if (config_fill && opt_quarantine)
289 quarantine_alloc_hook();
290 }
291
292 JEMALLOC_ALWAYS_INLINE_C bool
malloc_init(void)293 malloc_init(void)
294 {
295
296 if (malloc_initialized == false && malloc_init_hard())
297 return (true);
298 malloc_thread_init();
299
300 return (false);
301 }
302
303 static bool
malloc_conf_next(char const ** opts_p,char const ** k_p,size_t * klen_p,char const ** v_p,size_t * vlen_p)304 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
305 char const **v_p, size_t *vlen_p)
306 {
307 bool accept;
308 const char *opts = *opts_p;
309
310 *k_p = opts;
311
312 for (accept = false; accept == false;) {
313 switch (*opts) {
314 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
315 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
316 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
317 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
318 case 'Y': case 'Z':
319 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
320 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
321 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
322 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
323 case 'y': case 'z':
324 case '0': case '1': case '2': case '3': case '4': case '5':
325 case '6': case '7': case '8': case '9':
326 case '_':
327 opts++;
328 break;
329 case ':':
330 opts++;
331 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
332 *v_p = opts;
333 accept = true;
334 break;
335 case '\0':
336 if (opts != *opts_p) {
337 malloc_write("<jemalloc>: Conf string ends "
338 "with key\n");
339 }
340 return (true);
341 default:
342 malloc_write("<jemalloc>: Malformed conf string\n");
343 return (true);
344 }
345 }
346
347 for (accept = false; accept == false;) {
348 switch (*opts) {
349 case ',':
350 opts++;
351 /*
352 * Look ahead one character here, because the next time
353 * this function is called, it will assume that end of
354 * input has been cleanly reached if no input remains,
355 * but we have optimistically already consumed the
356 * comma if one exists.
357 */
358 if (*opts == '\0') {
359 malloc_write("<jemalloc>: Conf string ends "
360 "with comma\n");
361 }
362 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
363 accept = true;
364 break;
365 case '\0':
366 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
367 accept = true;
368 break;
369 default:
370 opts++;
371 break;
372 }
373 }
374
375 *opts_p = opts;
376 return (false);
377 }
378
379 static void
malloc_conf_error(const char * msg,const char * k,size_t klen,const char * v,size_t vlen)380 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
381 size_t vlen)
382 {
383
384 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
385 (int)vlen, v);
386 }
387
388 static void
malloc_conf_init(void)389 malloc_conf_init(void)
390 {
391 unsigned i;
392 char buf[PATH_MAX + 1];
393 const char *opts, *k, *v;
394 size_t klen, vlen;
395
396 /*
397 * Automatically configure valgrind before processing options. The
398 * valgrind option remains in jemalloc 3.x for compatibility reasons.
399 */
400 if (config_valgrind) {
401 opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
402 if (config_fill && opt_valgrind) {
403 opt_junk = false;
404 assert(opt_zero == false);
405 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
406 opt_redzone = true;
407 }
408 if (config_tcache && opt_valgrind)
409 opt_tcache = false;
410 }
411
412 for (i = 0; i < 3; i++) {
413 /* Get runtime configuration. */
414 switch (i) {
415 case 0:
416 if (je_malloc_conf != NULL) {
417 /*
418 * Use options that were compiled into the
419 * program.
420 */
421 opts = je_malloc_conf;
422 } else {
423 /* No configuration specified. */
424 buf[0] = '\0';
425 opts = buf;
426 }
427 break;
428 case 1: {
429 int linklen = 0;
430 #ifndef _WIN32
431 int saved_errno = errno;
432 const char *linkname =
433 # ifdef JEMALLOC_PREFIX
434 "/etc/"JEMALLOC_PREFIX"malloc.conf"
435 # else
436 "/etc/malloc.conf"
437 # endif
438 ;
439
440 /*
441 * Try to use the contents of the "/etc/malloc.conf"
442 * symbolic link's name.
443 */
444 linklen = readlink(linkname, buf, sizeof(buf) - 1);
445 if (linklen == -1) {
446 /* No configuration specified. */
447 linklen = 0;
448 /* restore errno */
449 set_errno(saved_errno);
450 }
451 #endif
452 buf[linklen] = '\0';
453 opts = buf;
454 break;
455 } case 2: {
456 const char *envname =
457 #ifdef JEMALLOC_PREFIX
458 JEMALLOC_CPREFIX"MALLOC_CONF"
459 #else
460 "MALLOC_CONF"
461 #endif
462 ;
463
464 if (issetugid() == 0 && (opts = getenv(envname)) !=
465 NULL) {
466 /*
467 * Do nothing; opts is already initialized to
468 * the value of the MALLOC_CONF environment
469 * variable.
470 */
471 } else {
472 /* No configuration specified. */
473 buf[0] = '\0';
474 opts = buf;
475 }
476 break;
477 } default:
478 not_reached();
479 buf[0] = '\0';
480 opts = buf;
481 }
482
483 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
484 &vlen) == false) {
485 #define CONF_HANDLE_BOOL(o, n) \
486 if (sizeof(n)-1 == klen && strncmp(n, k, \
487 klen) == 0) { \
488 if (strncmp("true", v, vlen) == 0 && \
489 vlen == sizeof("true")-1) \
490 o = true; \
491 else if (strncmp("false", v, vlen) == \
492 0 && vlen == sizeof("false")-1) \
493 o = false; \
494 else { \
495 malloc_conf_error( \
496 "Invalid conf value", \
497 k, klen, v, vlen); \
498 } \
499 continue; \
500 }
501 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \
502 if (sizeof(n)-1 == klen && strncmp(n, k, \
503 klen) == 0) { \
504 uintmax_t um; \
505 char *end; \
506 \
507 set_errno(0); \
508 um = malloc_strtoumax(v, &end, 0); \
509 if (get_errno() != 0 || (uintptr_t)end -\
510 (uintptr_t)v != vlen) { \
511 malloc_conf_error( \
512 "Invalid conf value", \
513 k, klen, v, vlen); \
514 } else if (clip) { \
515 if (min != 0 && um < min) \
516 o = min; \
517 else if (um > max) \
518 o = max; \
519 else \
520 o = um; \
521 } else { \
522 if ((min != 0 && um < min) || \
523 um > max) { \
524 malloc_conf_error( \
525 "Out-of-range " \
526 "conf value", \
527 k, klen, v, vlen); \
528 } else \
529 o = um; \
530 } \
531 continue; \
532 }
533 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \
534 if (sizeof(n)-1 == klen && strncmp(n, k, \
535 klen) == 0) { \
536 long l; \
537 char *end; \
538 \
539 set_errno(0); \
540 l = strtol(v, &end, 0); \
541 if (get_errno() != 0 || (uintptr_t)end -\
542 (uintptr_t)v != vlen) { \
543 malloc_conf_error( \
544 "Invalid conf value", \
545 k, klen, v, vlen); \
546 } else if (l < (ssize_t)min || l > \
547 (ssize_t)max) { \
548 malloc_conf_error( \
549 "Out-of-range conf value", \
550 k, klen, v, vlen); \
551 } else \
552 o = l; \
553 continue; \
554 }
555 #define CONF_HANDLE_CHAR_P(o, n, d) \
556 if (sizeof(n)-1 == klen && strncmp(n, k, \
557 klen) == 0) { \
558 size_t cpylen = (vlen <= \
559 sizeof(o)-1) ? vlen : \
560 sizeof(o)-1; \
561 strncpy(o, v, cpylen); \
562 o[cpylen] = '\0'; \
563 continue; \
564 }
565
566 CONF_HANDLE_BOOL(opt_abort, "abort")
567 /*
568 * Chunks always require at least one header page, plus
569 * one data page in the absence of redzones, or three
570 * pages in the presence of redzones. In order to
571 * simplify options processing, fix the limit based on
572 * config_fill.
573 */
574 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
575 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
576 true)
577 if (strncmp("dss", k, klen) == 0) {
578 int i;
579 bool match = false;
580 for (i = 0; i < dss_prec_limit; i++) {
581 if (strncmp(dss_prec_names[i], v, vlen)
582 == 0) {
583 if (chunk_dss_prec_set(i)) {
584 malloc_conf_error(
585 "Error setting dss",
586 k, klen, v, vlen);
587 } else {
588 opt_dss =
589 dss_prec_names[i];
590 match = true;
591 break;
592 }
593 }
594 }
595 if (match == false) {
596 malloc_conf_error("Invalid conf value",
597 k, klen, v, vlen);
598 }
599 continue;
600 }
601 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
602 SIZE_T_MAX, false)
603 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
604 -1, (sizeof(size_t) << 3) - 1)
605 CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
606 if (config_fill) {
607 CONF_HANDLE_BOOL(opt_junk, "junk")
608 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
609 0, SIZE_T_MAX, false)
610 CONF_HANDLE_BOOL(opt_redzone, "redzone")
611 CONF_HANDLE_BOOL(opt_zero, "zero")
612 }
613 if (config_utrace) {
614 CONF_HANDLE_BOOL(opt_utrace, "utrace")
615 }
616 if (config_valgrind) {
617 CONF_HANDLE_BOOL(opt_valgrind, "valgrind")
618 }
619 if (config_xmalloc) {
620 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
621 }
622 if (config_tcache) {
623 CONF_HANDLE_BOOL(opt_tcache, "tcache")
624 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
625 "lg_tcache_max", -1,
626 (sizeof(size_t) << 3) - 1)
627 }
628 if (config_prof) {
629 CONF_HANDLE_BOOL(opt_prof, "prof")
630 CONF_HANDLE_CHAR_P(opt_prof_prefix,
631 "prof_prefix", "jeprof")
632 CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
633 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
634 "lg_prof_sample", 0,
635 (sizeof(uint64_t) << 3) - 1)
636 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
637 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
638 "lg_prof_interval", -1,
639 (sizeof(uint64_t) << 3) - 1)
640 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
641 CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
642 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
643 }
644 malloc_conf_error("Invalid conf pair", k, klen, v,
645 vlen);
646 #undef CONF_HANDLE_BOOL
647 #undef CONF_HANDLE_SIZE_T
648 #undef CONF_HANDLE_SSIZE_T
649 #undef CONF_HANDLE_CHAR_P
650 }
651 }
652 }
653
654 static bool
malloc_init_hard(void)655 malloc_init_hard(void)
656 {
657 arena_t *init_arenas[1];
658
659 malloc_mutex_lock(&init_lock);
660 if (malloc_initialized || IS_INITIALIZER) {
661 /*
662 * Another thread initialized the allocator before this one
663 * acquired init_lock, or this thread is the initializing
664 * thread, and it is recursively allocating.
665 */
666 malloc_mutex_unlock(&init_lock);
667 return (false);
668 }
669 #ifdef JEMALLOC_THREADED_INIT
670 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) {
671 /* Busy-wait until the initializing thread completes. */
672 do {
673 malloc_mutex_unlock(&init_lock);
674 CPU_SPINWAIT;
675 malloc_mutex_lock(&init_lock);
676 } while (malloc_initialized == false);
677 malloc_mutex_unlock(&init_lock);
678 return (false);
679 }
680 #endif
681 malloc_initializer = INITIALIZER;
682
683 malloc_tsd_boot();
684 if (config_prof)
685 prof_boot0();
686
687 malloc_conf_init();
688
689 if (opt_stats_print) {
690 /* Print statistics at exit. */
691 if (atexit(stats_print_atexit) != 0) {
692 malloc_write("<jemalloc>: Error in atexit()\n");
693 if (opt_abort)
694 abort();
695 }
696 }
697
698 if (base_boot()) {
699 malloc_mutex_unlock(&init_lock);
700 return (true);
701 }
702
703 if (chunk_boot()) {
704 malloc_mutex_unlock(&init_lock);
705 return (true);
706 }
707
708 if (ctl_boot()) {
709 malloc_mutex_unlock(&init_lock);
710 return (true);
711 }
712
713 if (config_prof)
714 prof_boot1();
715
716 arena_boot();
717
718 if (config_tcache && tcache_boot0()) {
719 malloc_mutex_unlock(&init_lock);
720 return (true);
721 }
722
723 if (huge_boot()) {
724 malloc_mutex_unlock(&init_lock);
725 return (true);
726 }
727
728 if (malloc_mutex_init(&arenas_lock)) {
729 malloc_mutex_unlock(&init_lock);
730 return (true);
731 }
732
733 /*
734 * Create enough scaffolding to allow recursive allocation in
735 * malloc_ncpus().
736 */
737 narenas_total = narenas_auto = 1;
738 arenas = init_arenas;
739 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
740
741 /*
742 * Initialize one arena here. The rest are lazily created in
743 * choose_arena_hard().
744 */
745 arenas_extend(0);
746 if (arenas[0] == NULL) {
747 malloc_mutex_unlock(&init_lock);
748 return (true);
749 }
750
751 /* Initialize allocation counters before any allocations can occur. */
752 if (config_stats && thread_allocated_tsd_boot()) {
753 malloc_mutex_unlock(&init_lock);
754 return (true);
755 }
756
757 if (arenas_tsd_boot()) {
758 malloc_mutex_unlock(&init_lock);
759 return (true);
760 }
761
762 if (config_tcache && tcache_boot1()) {
763 malloc_mutex_unlock(&init_lock);
764 return (true);
765 }
766
767 if (config_fill && quarantine_boot()) {
768 malloc_mutex_unlock(&init_lock);
769 return (true);
770 }
771
772 if (config_prof && prof_boot2()) {
773 malloc_mutex_unlock(&init_lock);
774 return (true);
775 }
776
777 malloc_mutex_unlock(&init_lock);
778 /**********************************************************************/
779 /* Recursive allocation may follow. */
780
781 ncpus = malloc_ncpus();
782
783 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
784 && !defined(_WIN32))
785 /* LinuxThreads's pthread_atfork() allocates. */
786 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
787 jemalloc_postfork_child) != 0) {
788 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
789 if (opt_abort)
790 abort();
791 }
792 #endif
793
794 /* Done recursively allocating. */
795 /**********************************************************************/
796 malloc_mutex_lock(&init_lock);
797
798 if (mutex_boot()) {
799 malloc_mutex_unlock(&init_lock);
800 return (true);
801 }
802
803 if (opt_narenas == 0) {
804 /*
805 * For SMP systems, create more than one arena per CPU by
806 * default.
807 */
808 if (ncpus > 1)
809 opt_narenas = ncpus << 2;
810 else
811 opt_narenas = 1;
812 }
813 narenas_auto = opt_narenas;
814 /*
815 * Make sure that the arenas array can be allocated. In practice, this
816 * limit is enough to allow the allocator to function, but the ctl
817 * machinery will fail to allocate memory at far lower limits.
818 */
819 if (narenas_auto > chunksize / sizeof(arena_t *)) {
820 narenas_auto = chunksize / sizeof(arena_t *);
821 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
822 narenas_auto);
823 }
824 narenas_total = narenas_auto;
825
826 /* Allocate and initialize arenas. */
827 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
828 if (arenas == NULL) {
829 malloc_mutex_unlock(&init_lock);
830 return (true);
831 }
832 /*
833 * Zero the array. In practice, this should always be pre-zeroed,
834 * since it was just mmap()ed, but let's be sure.
835 */
836 memset(arenas, 0, sizeof(arena_t *) * narenas_total);
837 /* Copy the pointer to the one arena that was already initialized. */
838 arenas[0] = init_arenas[0];
839
840 malloc_initialized = true;
841 malloc_mutex_unlock(&init_lock);
842
843 return (false);
844 }
845
846 /*
847 * End initialization functions.
848 */
849 /******************************************************************************/
850 /*
851 * Begin malloc(3)-compatible functions.
852 */
853
854 static void *
imalloc_prof_sample(size_t usize,prof_thr_cnt_t * cnt)855 imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
856 {
857 void *p;
858
859 if (cnt == NULL)
860 return (NULL);
861 if (prof_promote && usize <= SMALL_MAXCLASS) {
862 p = imalloc(SMALL_MAXCLASS+1);
863 if (p == NULL)
864 return (NULL);
865 arena_prof_promoted(p, usize);
866 } else
867 p = imalloc(usize);
868
869 return (p);
870 }
871
872 JEMALLOC_ALWAYS_INLINE_C void *
imalloc_prof(size_t usize,prof_thr_cnt_t * cnt)873 imalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
874 {
875 void *p;
876
877 if ((uintptr_t)cnt != (uintptr_t)1U)
878 p = imalloc_prof_sample(usize, cnt);
879 else
880 p = imalloc(usize);
881 if (p == NULL)
882 return (NULL);
883 prof_malloc(p, usize, cnt);
884
885 return (p);
886 }
887
888 /*
889 * MALLOC_BODY() is a macro rather than a function because its contents are in
890 * the fast path, but inlining would cause reliability issues when determining
891 * how many frames to discard from heap profiling backtraces.
892 */
893 #define MALLOC_BODY(ret, size, usize) do { \
894 if (malloc_init()) \
895 ret = NULL; \
896 else { \
897 if (config_prof && opt_prof) { \
898 prof_thr_cnt_t *cnt; \
899 \
900 usize = s2u(size); \
901 /* \
902 * Call PROF_ALLOC_PREP() here rather than in \
903 * imalloc_prof() so that imalloc_prof() can be \
904 * inlined without introducing uncertainty \
905 * about the number of backtrace frames to \
906 * ignore. imalloc_prof() is in the fast path \
907 * when heap profiling is enabled, so inlining \
908 * is critical to performance. (For \
909 * consistency all callers of PROF_ALLOC_PREP() \
910 * are structured similarly, even though e.g. \
911 * realloc() isn't called enough for inlining \
912 * to be critical.) \
913 */ \
914 PROF_ALLOC_PREP(1, usize, cnt); \
915 ret = imalloc_prof(usize, cnt); \
916 } else { \
917 if (config_stats || (config_valgrind && \
918 opt_valgrind)) \
919 usize = s2u(size); \
920 ret = imalloc(size); \
921 } \
922 } \
923 } while (0)
924
925 void *
je_malloc(size_t size)926 je_malloc(size_t size)
927 {
928 void *ret;
929 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
930
931 if (size == 0)
932 size = 1;
933
934 MALLOC_BODY(ret, size, usize);
935
936 if (ret == NULL) {
937 if (config_xmalloc && opt_xmalloc) {
938 malloc_write("<jemalloc>: Error in malloc(): "
939 "out of memory\n");
940 abort();
941 }
942 set_errno(ENOMEM);
943 }
944 if (config_stats && ret != NULL) {
945 assert(usize == isalloc(ret, config_prof));
946 thread_allocated_tsd_get()->allocated += usize;
947 }
948 UTRACE(0, size, ret);
949 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
950 return (ret);
951 }
952
953 static void *
imemalign_prof_sample(size_t alignment,size_t usize,prof_thr_cnt_t * cnt)954 imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
955 {
956 void *p;
957
958 if (cnt == NULL)
959 return (NULL);
960 if (prof_promote && usize <= SMALL_MAXCLASS) {
961 assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0);
962 p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment,
963 false);
964 if (p == NULL)
965 return (NULL);
966 arena_prof_promoted(p, usize);
967 } else
968 p = ipalloc(usize, alignment, false);
969
970 return (p);
971 }
972
973 JEMALLOC_ALWAYS_INLINE_C void *
imemalign_prof(size_t alignment,size_t usize,prof_thr_cnt_t * cnt)974 imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
975 {
976 void *p;
977
978 if ((uintptr_t)cnt != (uintptr_t)1U)
979 p = imemalign_prof_sample(alignment, usize, cnt);
980 else
981 p = ipalloc(usize, alignment, false);
982 if (p == NULL)
983 return (NULL);
984 prof_malloc(p, usize, cnt);
985
986 return (p);
987 }
988
989 JEMALLOC_ATTR(nonnull(1))
990 #ifdef JEMALLOC_PROF
991 /*
992 * Avoid any uncertainty as to how many backtrace frames to ignore in
993 * PROF_ALLOC_PREP().
994 */
995 JEMALLOC_NOINLINE
996 #endif
997 static int
imemalign(void ** memptr,size_t alignment,size_t size,size_t min_alignment)998 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
999 {
1000 int ret;
1001 size_t usize;
1002 void *result;
1003
1004 assert(min_alignment != 0);
1005
1006 if (malloc_init()) {
1007 result = NULL;
1008 goto label_oom;
1009 } else {
1010 if (size == 0)
1011 size = 1;
1012
1013 /* Make sure that alignment is a large enough power of 2. */
1014 if (((alignment - 1) & alignment) != 0
1015 || (alignment < min_alignment)) {
1016 if (config_xmalloc && opt_xmalloc) {
1017 malloc_write("<jemalloc>: Error allocating "
1018 "aligned memory: invalid alignment\n");
1019 abort();
1020 }
1021 result = NULL;
1022 ret = EINVAL;
1023 goto label_return;
1024 }
1025
1026 usize = sa2u(size, alignment);
1027 if (usize == 0) {
1028 result = NULL;
1029 goto label_oom;
1030 }
1031
1032 if (config_prof && opt_prof) {
1033 prof_thr_cnt_t *cnt;
1034
1035 PROF_ALLOC_PREP(2, usize, cnt);
1036 result = imemalign_prof(alignment, usize, cnt);
1037 } else
1038 result = ipalloc(usize, alignment, false);
1039 if (result == NULL)
1040 goto label_oom;
1041 }
1042
1043 *memptr = result;
1044 ret = 0;
1045 label_return:
1046 if (config_stats && result != NULL) {
1047 assert(usize == isalloc(result, config_prof));
1048 thread_allocated_tsd_get()->allocated += usize;
1049 }
1050 UTRACE(0, size, result);
1051 return (ret);
1052 label_oom:
1053 assert(result == NULL);
1054 if (config_xmalloc && opt_xmalloc) {
1055 malloc_write("<jemalloc>: Error allocating aligned memory: "
1056 "out of memory\n");
1057 abort();
1058 }
1059 ret = ENOMEM;
1060 goto label_return;
1061 }
1062
1063 int
je_posix_memalign(void ** memptr,size_t alignment,size_t size)1064 je_posix_memalign(void **memptr, size_t alignment, size_t size)
1065 {
1066 int ret = imemalign(memptr, alignment, size, sizeof(void *));
1067 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
1068 config_prof), false);
1069 return (ret);
1070 }
1071
1072 void *
je_aligned_alloc(size_t alignment,size_t size)1073 je_aligned_alloc(size_t alignment, size_t size)
1074 {
1075 void *ret;
1076 int err;
1077
1078 if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
1079 ret = NULL;
1080 set_errno(err);
1081 }
1082 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
1083 false);
1084 return (ret);
1085 }
1086
1087 static void *
icalloc_prof_sample(size_t usize,prof_thr_cnt_t * cnt)1088 icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
1089 {
1090 void *p;
1091
1092 if (cnt == NULL)
1093 return (NULL);
1094 if (prof_promote && usize <= SMALL_MAXCLASS) {
1095 p = icalloc(SMALL_MAXCLASS+1);
1096 if (p == NULL)
1097 return (NULL);
1098 arena_prof_promoted(p, usize);
1099 } else
1100 p = icalloc(usize);
1101
1102 return (p);
1103 }
1104
1105 JEMALLOC_ALWAYS_INLINE_C void *
icalloc_prof(size_t usize,prof_thr_cnt_t * cnt)1106 icalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
1107 {
1108 void *p;
1109
1110 if ((uintptr_t)cnt != (uintptr_t)1U)
1111 p = icalloc_prof_sample(usize, cnt);
1112 else
1113 p = icalloc(usize);
1114 if (p == NULL)
1115 return (NULL);
1116 prof_malloc(p, usize, cnt);
1117
1118 return (p);
1119 }
1120
1121 void *
je_calloc(size_t num,size_t size)1122 je_calloc(size_t num, size_t size)
1123 {
1124 void *ret;
1125 size_t num_size;
1126 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1127
1128 if (malloc_init()) {
1129 num_size = 0;
1130 ret = NULL;
1131 goto label_return;
1132 }
1133
1134 num_size = num * size;
1135 if (num_size == 0) {
1136 if (num == 0 || size == 0)
1137 num_size = 1;
1138 else {
1139 ret = NULL;
1140 goto label_return;
1141 }
1142 /*
1143 * Try to avoid division here. We know that it isn't possible to
1144 * overflow during multiplication if neither operand uses any of the
1145 * most significant half of the bits in a size_t.
1146 */
1147 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1148 && (num_size / size != num)) {
1149 /* size_t overflow. */
1150 ret = NULL;
1151 goto label_return;
1152 }
1153
1154 if (config_prof && opt_prof) {
1155 prof_thr_cnt_t *cnt;
1156
1157 usize = s2u(num_size);
1158 PROF_ALLOC_PREP(1, usize, cnt);
1159 ret = icalloc_prof(usize, cnt);
1160 } else {
1161 if (config_stats || (config_valgrind && opt_valgrind))
1162 usize = s2u(num_size);
1163 ret = icalloc(num_size);
1164 }
1165
1166 label_return:
1167 if (ret == NULL) {
1168 if (config_xmalloc && opt_xmalloc) {
1169 malloc_write("<jemalloc>: Error in calloc(): out of "
1170 "memory\n");
1171 abort();
1172 }
1173 set_errno(ENOMEM);
1174 }
1175 if (config_stats && ret != NULL) {
1176 assert(usize == isalloc(ret, config_prof));
1177 thread_allocated_tsd_get()->allocated += usize;
1178 }
1179 UTRACE(0, num_size, ret);
1180 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
1181 return (ret);
1182 }
1183
1184 static void *
irealloc_prof_sample(void * oldptr,size_t usize,prof_thr_cnt_t * cnt)1185 irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
1186 {
1187 void *p;
1188
1189 if (cnt == NULL)
1190 return (NULL);
1191 if (prof_promote && usize <= SMALL_MAXCLASS) {
1192 p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false);
1193 if (p == NULL)
1194 return (NULL);
1195 arena_prof_promoted(p, usize);
1196 } else
1197 p = iralloc(oldptr, usize, 0, 0, false);
1198
1199 return (p);
1200 }
1201
1202 JEMALLOC_ALWAYS_INLINE_C void *
irealloc_prof(void * oldptr,size_t old_usize,size_t usize,prof_thr_cnt_t * cnt)1203 irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt)
1204 {
1205 void *p;
1206 prof_ctx_t *old_ctx;
1207
1208 old_ctx = prof_ctx_get(oldptr);
1209 if ((uintptr_t)cnt != (uintptr_t)1U)
1210 p = irealloc_prof_sample(oldptr, usize, cnt);
1211 else
1212 p = iralloc(oldptr, usize, 0, 0, false);
1213 if (p == NULL)
1214 return (NULL);
1215 prof_realloc(p, usize, cnt, old_usize, old_ctx);
1216
1217 return (p);
1218 }
1219
1220 JEMALLOC_INLINE_C void
ifree(void * ptr)1221 ifree(void *ptr)
1222 {
1223 size_t usize;
1224 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1225
1226 assert(ptr != NULL);
1227 assert(malloc_initialized || IS_INITIALIZER);
1228
1229 if (config_prof && opt_prof) {
1230 usize = isalloc(ptr, config_prof);
1231 prof_free(ptr, usize);
1232 } else if (config_stats || config_valgrind)
1233 usize = isalloc(ptr, config_prof);
1234 if (config_stats)
1235 thread_allocated_tsd_get()->deallocated += usize;
1236 if (config_valgrind && opt_valgrind)
1237 rzsize = p2rz(ptr);
1238 iqalloc(ptr);
1239 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1240 }
1241
1242 void *
je_realloc(void * ptr,size_t size)1243 je_realloc(void *ptr, size_t size)
1244 {
1245 void *ret;
1246 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1247 size_t old_usize = 0;
1248 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1249
1250 if (size == 0) {
1251 if (ptr != NULL) {
1252 /* realloc(ptr, 0) is equivalent to free(ptr). */
1253 UTRACE(ptr, 0, 0);
1254 ifree(ptr);
1255 return (NULL);
1256 }
1257 size = 1;
1258 }
1259
1260 if (ptr != NULL) {
1261 assert(malloc_initialized || IS_INITIALIZER);
1262 malloc_thread_init();
1263
1264 if ((config_prof && opt_prof) || config_stats ||
1265 (config_valgrind && opt_valgrind))
1266 old_usize = isalloc(ptr, config_prof);
1267 if (config_valgrind && opt_valgrind)
1268 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
1269
1270 if (config_prof && opt_prof) {
1271 prof_thr_cnt_t *cnt;
1272
1273 usize = s2u(size);
1274 PROF_ALLOC_PREP(1, usize, cnt);
1275 ret = irealloc_prof(ptr, old_usize, usize, cnt);
1276 } else {
1277 if (config_stats || (config_valgrind && opt_valgrind))
1278 usize = s2u(size);
1279 ret = iralloc(ptr, size, 0, 0, false);
1280 }
1281 } else {
1282 /* realloc(NULL, size) is equivalent to malloc(size). */
1283 MALLOC_BODY(ret, size, usize);
1284 }
1285
1286 if (ret == NULL) {
1287 if (config_xmalloc && opt_xmalloc) {
1288 malloc_write("<jemalloc>: Error in realloc(): "
1289 "out of memory\n");
1290 abort();
1291 }
1292 set_errno(ENOMEM);
1293 }
1294 if (config_stats && ret != NULL) {
1295 thread_allocated_t *ta;
1296 assert(usize == isalloc(ret, config_prof));
1297 ta = thread_allocated_tsd_get();
1298 ta->allocated += usize;
1299 ta->deallocated += old_usize;
1300 }
1301 UTRACE(ptr, size, ret);
1302 JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize,
1303 false);
1304 return (ret);
1305 }
1306
1307 void
je_free(void * ptr)1308 je_free(void *ptr)
1309 {
1310
1311 UTRACE(ptr, 0, 0);
1312 if (ptr != NULL)
1313 ifree(ptr);
1314 }
1315
1316 /*
1317 * End malloc(3)-compatible functions.
1318 */
1319 /******************************************************************************/
1320 /*
1321 * Begin non-standard override functions.
1322 */
1323
1324 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
1325 void *
je_memalign(size_t alignment,size_t size)1326 je_memalign(size_t alignment, size_t size)
1327 {
1328 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1329 imemalign(&ret, alignment, size, 1);
1330 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1331 return (ret);
1332 }
1333 #endif
1334
1335 #ifdef JEMALLOC_OVERRIDE_VALLOC
1336 void *
je_valloc(size_t size)1337 je_valloc(size_t size)
1338 {
1339 void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1340 imemalign(&ret, PAGE, size, 1);
1341 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1342 return (ret);
1343 }
1344 #endif
1345
1346 /*
1347 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1348 * #define je_malloc malloc
1349 */
1350 #define malloc_is_malloc 1
1351 #define is_malloc_(a) malloc_is_ ## a
1352 #define is_malloc(a) is_malloc_(a)
1353
1354 #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__))
1355 /*
1356 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1357 * to inconsistently reference libc's malloc(3)-compatible functions
1358 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1359 *
1360 * These definitions interpose hooks in glibc. The functions are actually
1361 * passed an extra argument for the caller return address, which will be
1362 * ignored.
1363 */
1364 JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free;
1365 JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc;
1366 JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc;
1367 JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) =
1368 je_memalign;
1369 #endif
1370
1371 /*
1372 * End non-standard override functions.
1373 */
1374 /******************************************************************************/
1375 /*
1376 * Begin non-standard functions.
1377 */
1378
1379 JEMALLOC_ALWAYS_INLINE_C void *
imallocx(size_t usize,size_t alignment,bool zero,bool try_tcache,arena_t * arena)1380 imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
1381 arena_t *arena)
1382 {
1383
1384 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
1385 alignment)));
1386
1387 if (alignment != 0)
1388 return (ipalloct(usize, alignment, zero, try_tcache, arena));
1389 else if (zero)
1390 return (icalloct(usize, try_tcache, arena));
1391 else
1392 return (imalloct(usize, try_tcache, arena));
1393 }
1394
1395 static void *
imallocx_prof_sample(size_t usize,size_t alignment,bool zero,bool try_tcache,arena_t * arena,prof_thr_cnt_t * cnt)1396 imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
1397 arena_t *arena, prof_thr_cnt_t *cnt)
1398 {
1399 void *p;
1400
1401 if (cnt == NULL)
1402 return (NULL);
1403 if (prof_promote && usize <= SMALL_MAXCLASS) {
1404 size_t usize_promoted = (alignment == 0) ?
1405 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment);
1406 assert(usize_promoted != 0);
1407 p = imallocx(usize_promoted, alignment, zero, try_tcache,
1408 arena);
1409 if (p == NULL)
1410 return (NULL);
1411 arena_prof_promoted(p, usize);
1412 } else
1413 p = imallocx(usize, alignment, zero, try_tcache, arena);
1414
1415 return (p);
1416 }
1417
1418 JEMALLOC_ALWAYS_INLINE_C void *
imallocx_prof(size_t usize,size_t alignment,bool zero,bool try_tcache,arena_t * arena,prof_thr_cnt_t * cnt)1419 imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache,
1420 arena_t *arena, prof_thr_cnt_t *cnt)
1421 {
1422 void *p;
1423
1424 if ((uintptr_t)cnt != (uintptr_t)1U) {
1425 p = imallocx_prof_sample(usize, alignment, zero, try_tcache,
1426 arena, cnt);
1427 } else
1428 p = imallocx(usize, alignment, zero, try_tcache, arena);
1429 if (p == NULL)
1430 return (NULL);
1431 prof_malloc(p, usize, cnt);
1432
1433 return (p);
1434 }
1435
1436 void *
je_mallocx(size_t size,int flags)1437 je_mallocx(size_t size, int flags)
1438 {
1439 void *p;
1440 size_t usize;
1441 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1442 & (SIZE_T_MAX-1));
1443 bool zero = flags & MALLOCX_ZERO;
1444 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1445 arena_t *arena;
1446 bool try_tcache;
1447
1448 assert(size != 0);
1449
1450 if (malloc_init())
1451 goto label_oom;
1452
1453 if (arena_ind != UINT_MAX) {
1454 arena = arenas[arena_ind];
1455 try_tcache = false;
1456 } else {
1457 arena = NULL;
1458 try_tcache = true;
1459 }
1460
1461 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1462 assert(usize != 0);
1463
1464 if (config_prof && opt_prof) {
1465 prof_thr_cnt_t *cnt;
1466
1467 PROF_ALLOC_PREP(1, usize, cnt);
1468 p = imallocx_prof(usize, alignment, zero, try_tcache, arena,
1469 cnt);
1470 } else
1471 p = imallocx(usize, alignment, zero, try_tcache, arena);
1472 if (p == NULL)
1473 goto label_oom;
1474
1475 if (config_stats) {
1476 assert(usize == isalloc(p, config_prof));
1477 thread_allocated_tsd_get()->allocated += usize;
1478 }
1479 UTRACE(0, size, p);
1480 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
1481 return (p);
1482 label_oom:
1483 if (config_xmalloc && opt_xmalloc) {
1484 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n");
1485 abort();
1486 }
1487 UTRACE(0, size, 0);
1488 return (NULL);
1489 }
1490
1491 static void *
irallocx_prof_sample(void * oldptr,size_t size,size_t alignment,size_t usize,bool zero,bool try_tcache_alloc,bool try_tcache_dalloc,arena_t * arena,prof_thr_cnt_t * cnt)1492 irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
1493 bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
1494 prof_thr_cnt_t *cnt)
1495 {
1496 void *p;
1497
1498 if (cnt == NULL)
1499 return (NULL);
1500 if (prof_promote && usize <= SMALL_MAXCLASS) {
1501 p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1502 size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero,
1503 try_tcache_alloc, try_tcache_dalloc, arena);
1504 if (p == NULL)
1505 return (NULL);
1506 arena_prof_promoted(p, usize);
1507 } else {
1508 p = iralloct(oldptr, size, 0, alignment, zero,
1509 try_tcache_alloc, try_tcache_dalloc, arena);
1510 }
1511
1512 return (p);
1513 }
1514
1515 JEMALLOC_ALWAYS_INLINE_C void *
irallocx_prof(void * oldptr,size_t old_usize,size_t size,size_t alignment,size_t * usize,bool zero,bool try_tcache_alloc,bool try_tcache_dalloc,arena_t * arena,prof_thr_cnt_t * cnt)1516 irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
1517 size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
1518 arena_t *arena, prof_thr_cnt_t *cnt)
1519 {
1520 void *p;
1521 prof_ctx_t *old_ctx;
1522
1523 old_ctx = prof_ctx_get(oldptr);
1524 if ((uintptr_t)cnt != (uintptr_t)1U)
1525 p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
1526 try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1527 else {
1528 p = iralloct(oldptr, size, 0, alignment, zero,
1529 try_tcache_alloc, try_tcache_dalloc, arena);
1530 }
1531 if (p == NULL)
1532 return (NULL);
1533
1534 if (p == oldptr && alignment != 0) {
1535 /*
1536 * The allocation did not move, so it is possible that the size
1537 * class is smaller than would guarantee the requested
1538 * alignment, and that the alignment constraint was
1539 * serendipitously satisfied. Additionally, old_usize may not
1540 * be the same as the current usize because of in-place large
1541 * reallocation. Therefore, query the actual value of usize.
1542 */
1543 *usize = isalloc(p, config_prof);
1544 }
1545 prof_realloc(p, *usize, cnt, old_usize, old_ctx);
1546
1547 return (p);
1548 }
1549
1550 void *
je_rallocx(void * ptr,size_t size,int flags)1551 je_rallocx(void *ptr, size_t size, int flags)
1552 {
1553 void *p;
1554 size_t usize, old_usize;
1555 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1556 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1557 & (SIZE_T_MAX-1));
1558 bool zero = flags & MALLOCX_ZERO;
1559 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1560 bool try_tcache_alloc, try_tcache_dalloc;
1561 arena_t *arena;
1562
1563 assert(ptr != NULL);
1564 assert(size != 0);
1565 assert(malloc_initialized || IS_INITIALIZER);
1566 malloc_thread_init();
1567
1568 if (arena_ind != UINT_MAX) {
1569 arena_chunk_t *chunk;
1570 try_tcache_alloc = false;
1571 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1572 try_tcache_dalloc = (chunk == ptr || chunk->arena !=
1573 arenas[arena_ind]);
1574 arena = arenas[arena_ind];
1575 } else {
1576 try_tcache_alloc = true;
1577 try_tcache_dalloc = true;
1578 arena = NULL;
1579 }
1580
1581 if ((config_prof && opt_prof) || config_stats ||
1582 (config_valgrind && opt_valgrind))
1583 old_usize = isalloc(ptr, config_prof);
1584 if (config_valgrind && opt_valgrind)
1585 old_rzsize = u2rz(old_usize);
1586
1587 if (config_prof && opt_prof) {
1588 prof_thr_cnt_t *cnt;
1589
1590 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1591 assert(usize != 0);
1592 PROF_ALLOC_PREP(1, usize, cnt);
1593 p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
1594 try_tcache_alloc, try_tcache_dalloc, arena, cnt);
1595 if (p == NULL)
1596 goto label_oom;
1597 } else {
1598 p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc,
1599 try_tcache_dalloc, arena);
1600 if (p == NULL)
1601 goto label_oom;
1602 if (config_stats || (config_valgrind && opt_valgrind))
1603 usize = isalloc(p, config_prof);
1604 }
1605
1606 if (config_stats) {
1607 thread_allocated_t *ta;
1608 ta = thread_allocated_tsd_get();
1609 ta->allocated += usize;
1610 ta->deallocated += old_usize;
1611 }
1612 UTRACE(ptr, size, p);
1613 JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero);
1614 return (p);
1615 label_oom:
1616 if (config_xmalloc && opt_xmalloc) {
1617 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
1618 abort();
1619 }
1620 UTRACE(ptr, size, 0);
1621 return (NULL);
1622 }
1623
1624 JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_helper(void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,bool zero,arena_t * arena)1625 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
1626 size_t alignment, bool zero, arena_t *arena)
1627 {
1628 size_t usize;
1629
1630 if (ixalloc(ptr, size, extra, alignment, zero))
1631 return (old_usize);
1632 usize = isalloc(ptr, config_prof);
1633
1634 return (usize);
1635 }
1636
1637 static size_t
ixallocx_prof_sample(void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,size_t max_usize,bool zero,arena_t * arena,prof_thr_cnt_t * cnt)1638 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
1639 size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1640 prof_thr_cnt_t *cnt)
1641 {
1642 size_t usize;
1643
1644 if (cnt == NULL)
1645 return (old_usize);
1646 /* Use minimum usize to determine whether promotion may happen. */
1647 if (prof_promote && ((alignment == 0) ? s2u(size) : sa2u(size,
1648 alignment)) <= SMALL_MAXCLASS) {
1649 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1650 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
1651 alignment, zero))
1652 return (old_usize);
1653 usize = isalloc(ptr, config_prof);
1654 if (max_usize < PAGE)
1655 arena_prof_promoted(ptr, usize);
1656 } else {
1657 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1658 zero, arena);
1659 }
1660
1661 return (usize);
1662 }
1663
1664 JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_prof(void * ptr,size_t old_usize,size_t size,size_t extra,size_t alignment,size_t max_usize,bool zero,arena_t * arena,prof_thr_cnt_t * cnt)1665 ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
1666 size_t alignment, size_t max_usize, bool zero, arena_t *arena,
1667 prof_thr_cnt_t *cnt)
1668 {
1669 size_t usize;
1670 prof_ctx_t *old_ctx;
1671
1672 old_ctx = prof_ctx_get(ptr);
1673 if ((uintptr_t)cnt != (uintptr_t)1U) {
1674 usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
1675 alignment, zero, max_usize, arena, cnt);
1676 } else {
1677 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1678 zero, arena);
1679 }
1680 if (usize == old_usize)
1681 return (usize);
1682 prof_realloc(ptr, usize, cnt, old_usize, old_ctx);
1683
1684 return (usize);
1685 }
1686
1687 size_t
je_xallocx(void * ptr,size_t size,size_t extra,int flags)1688 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
1689 {
1690 size_t usize, old_usize;
1691 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1692 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1693 & (SIZE_T_MAX-1));
1694 bool zero = flags & MALLOCX_ZERO;
1695 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1696 arena_t *arena;
1697
1698 assert(ptr != NULL);
1699 assert(size != 0);
1700 assert(SIZE_T_MAX - size >= extra);
1701 assert(malloc_initialized || IS_INITIALIZER);
1702 malloc_thread_init();
1703
1704 if (arena_ind != UINT_MAX)
1705 arena = arenas[arena_ind];
1706 else
1707 arena = NULL;
1708
1709 old_usize = isalloc(ptr, config_prof);
1710 if (config_valgrind && opt_valgrind)
1711 old_rzsize = u2rz(old_usize);
1712
1713 if (config_prof && opt_prof) {
1714 prof_thr_cnt_t *cnt;
1715 /*
1716 * usize isn't knowable before ixalloc() returns when extra is
1717 * non-zero. Therefore, compute its maximum possible value and
1718 * use that in PROF_ALLOC_PREP() to decide whether to capture a
1719 * backtrace. prof_realloc() will use the actual usize to
1720 * decide whether to sample.
1721 */
1722 size_t max_usize = (alignment == 0) ? s2u(size+extra) :
1723 sa2u(size+extra, alignment);
1724 PROF_ALLOC_PREP(1, max_usize, cnt);
1725 usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
1726 max_usize, zero, arena, cnt);
1727 } else {
1728 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
1729 zero, arena);
1730 }
1731 if (usize == old_usize)
1732 goto label_not_resized;
1733
1734 if (config_stats) {
1735 thread_allocated_t *ta;
1736 ta = thread_allocated_tsd_get();
1737 ta->allocated += usize;
1738 ta->deallocated += old_usize;
1739 }
1740 JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero);
1741 label_not_resized:
1742 UTRACE(ptr, size, ptr);
1743 return (usize);
1744 }
1745
1746 size_t
je_sallocx(const void * ptr,int flags)1747 je_sallocx(const void *ptr, int flags)
1748 {
1749 size_t usize;
1750
1751 assert(malloc_initialized || IS_INITIALIZER);
1752 malloc_thread_init();
1753
1754 if (config_ivsalloc)
1755 usize = ivsalloc(ptr, config_prof);
1756 else {
1757 assert(ptr != NULL);
1758 usize = isalloc(ptr, config_prof);
1759 }
1760
1761 return (usize);
1762 }
1763
1764 void
je_dallocx(void * ptr,int flags)1765 je_dallocx(void *ptr, int flags)
1766 {
1767 size_t usize;
1768 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1769 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1770 bool try_tcache;
1771
1772 assert(ptr != NULL);
1773 assert(malloc_initialized || IS_INITIALIZER);
1774
1775 if (arena_ind != UINT_MAX) {
1776 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1777 try_tcache = (chunk == ptr || chunk->arena !=
1778 arenas[arena_ind]);
1779 } else
1780 try_tcache = true;
1781
1782 UTRACE(ptr, 0, 0);
1783 if (config_stats || config_valgrind)
1784 usize = isalloc(ptr, config_prof);
1785 if (config_prof && opt_prof) {
1786 if (config_stats == false && config_valgrind == false)
1787 usize = isalloc(ptr, config_prof);
1788 prof_free(ptr, usize);
1789 }
1790 if (config_stats)
1791 thread_allocated_tsd_get()->deallocated += usize;
1792 if (config_valgrind && opt_valgrind)
1793 rzsize = p2rz(ptr);
1794 iqalloct(ptr, try_tcache);
1795 JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1796 }
1797
1798 size_t
je_nallocx(size_t size,int flags)1799 je_nallocx(size_t size, int flags)
1800 {
1801 size_t usize;
1802 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)
1803 & (SIZE_T_MAX-1));
1804
1805 assert(size != 0);
1806
1807 if (malloc_init())
1808 return (0);
1809
1810 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1811 assert(usize != 0);
1812 return (usize);
1813 }
1814
1815 int
je_mallctl(const char * name,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1816 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
1817 size_t newlen)
1818 {
1819
1820 if (malloc_init())
1821 return (EAGAIN);
1822
1823 return (ctl_byname(name, oldp, oldlenp, newp, newlen));
1824 }
1825
1826 int
je_mallctlnametomib(const char * name,size_t * mibp,size_t * miblenp)1827 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
1828 {
1829
1830 if (malloc_init())
1831 return (EAGAIN);
1832
1833 return (ctl_nametomib(name, mibp, miblenp));
1834 }
1835
1836 int
je_mallctlbymib(const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1837 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
1838 void *newp, size_t newlen)
1839 {
1840
1841 if (malloc_init())
1842 return (EAGAIN);
1843
1844 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
1845 }
1846
1847 void
je_malloc_stats_print(void (* write_cb)(void *,const char *),void * cbopaque,const char * opts)1848 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
1849 const char *opts)
1850 {
1851
1852 stats_print(write_cb, cbopaque, opts);
1853 }
1854
1855 size_t
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void * ptr)1856 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
1857 {
1858 size_t ret;
1859
1860 assert(malloc_initialized || IS_INITIALIZER);
1861 malloc_thread_init();
1862
1863 if (config_ivsalloc)
1864 ret = ivsalloc(ptr, config_prof);
1865 else
1866 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
1867
1868 return (ret);
1869 }
1870
1871 /*
1872 * End non-standard functions.
1873 */
1874 /******************************************************************************/
1875 /*
1876 * Begin experimental functions.
1877 */
1878 #ifdef JEMALLOC_EXPERIMENTAL
1879
1880 int
je_allocm(void ** ptr,size_t * rsize,size_t size,int flags)1881 je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
1882 {
1883 void *p;
1884
1885 assert(ptr != NULL);
1886
1887 p = je_mallocx(size, flags);
1888 if (p == NULL)
1889 return (ALLOCM_ERR_OOM);
1890 if (rsize != NULL)
1891 *rsize = isalloc(p, config_prof);
1892 *ptr = p;
1893 return (ALLOCM_SUCCESS);
1894 }
1895
1896 int
je_rallocm(void ** ptr,size_t * rsize,size_t size,size_t extra,int flags)1897 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
1898 {
1899 int ret;
1900 bool no_move = flags & ALLOCM_NO_MOVE;
1901
1902 assert(ptr != NULL);
1903 assert(*ptr != NULL);
1904 assert(size != 0);
1905 assert(SIZE_T_MAX - size >= extra);
1906
1907 if (no_move) {
1908 size_t usize = je_xallocx(*ptr, size, extra, flags);
1909 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
1910 if (rsize != NULL)
1911 *rsize = usize;
1912 } else {
1913 void *p = je_rallocx(*ptr, size+extra, flags);
1914 if (p != NULL) {
1915 *ptr = p;
1916 ret = ALLOCM_SUCCESS;
1917 } else
1918 ret = ALLOCM_ERR_OOM;
1919 if (rsize != NULL)
1920 *rsize = isalloc(*ptr, config_prof);
1921 }
1922 return (ret);
1923 }
1924
1925 int
je_sallocm(const void * ptr,size_t * rsize,int flags)1926 je_sallocm(const void *ptr, size_t *rsize, int flags)
1927 {
1928
1929 assert(rsize != NULL);
1930 *rsize = je_sallocx(ptr, flags);
1931 return (ALLOCM_SUCCESS);
1932 }
1933
1934 int
je_dallocm(void * ptr,int flags)1935 je_dallocm(void *ptr, int flags)
1936 {
1937
1938 je_dallocx(ptr, flags);
1939 return (ALLOCM_SUCCESS);
1940 }
1941
1942 int
je_nallocm(size_t * rsize,size_t size,int flags)1943 je_nallocm(size_t *rsize, size_t size, int flags)
1944 {
1945 size_t usize;
1946
1947 usize = je_nallocx(size, flags);
1948 if (usize == 0)
1949 return (ALLOCM_ERR_OOM);
1950 if (rsize != NULL)
1951 *rsize = usize;
1952 return (ALLOCM_SUCCESS);
1953 }
1954
1955 #endif
1956 /*
1957 * End experimental functions.
1958 */
1959 /******************************************************************************/
1960 /*
1961 * The following functions are used by threading libraries for protection of
1962 * malloc during fork().
1963 */
1964
1965 /*
1966 * If an application creates a thread before doing any allocation in the main
1967 * thread, then calls fork(2) in the main thread followed by memory allocation
1968 * in the child process, a race can occur that results in deadlock within the
1969 * child: the main thread may have forked while the created thread had
1970 * partially initialized the allocator. Ordinarily jemalloc prevents
1971 * fork/malloc races via the following functions it registers during
1972 * initialization using pthread_atfork(), but of course that does no good if
1973 * the allocator isn't fully initialized at fork time. The following library
1974 * constructor is a partial solution to this problem. It may still possible to
1975 * trigger the deadlock described above, but doing so would involve forking via
1976 * a library constructor that runs before jemalloc's runs.
1977 */
JEMALLOC_ATTR(constructor)1978 JEMALLOC_ATTR(constructor)
1979 static void
1980 jemalloc_constructor(void)
1981 {
1982
1983 malloc_init();
1984 }
1985
1986 #ifndef JEMALLOC_MUTEX_INIT_CB
1987 void
jemalloc_prefork(void)1988 jemalloc_prefork(void)
1989 #else
1990 JEMALLOC_EXPORT void
1991 _malloc_prefork(void)
1992 #endif
1993 {
1994 unsigned i;
1995
1996 #ifdef JEMALLOC_MUTEX_INIT_CB
1997 if (malloc_initialized == false)
1998 return;
1999 #endif
2000 assert(malloc_initialized);
2001
2002 /* Acquire all mutexes in a safe order. */
2003 ctl_prefork();
2004 prof_prefork();
2005 malloc_mutex_prefork(&arenas_lock);
2006 for (i = 0; i < narenas_total; i++) {
2007 if (arenas[i] != NULL)
2008 arena_prefork(arenas[i]);
2009 }
2010 chunk_prefork();
2011 base_prefork();
2012 huge_prefork();
2013 }
2014
2015 #ifndef JEMALLOC_MUTEX_INIT_CB
2016 void
jemalloc_postfork_parent(void)2017 jemalloc_postfork_parent(void)
2018 #else
2019 JEMALLOC_EXPORT void
2020 _malloc_postfork(void)
2021 #endif
2022 {
2023 unsigned i;
2024
2025 #ifdef JEMALLOC_MUTEX_INIT_CB
2026 if (malloc_initialized == false)
2027 return;
2028 #endif
2029 assert(malloc_initialized);
2030
2031 /* Release all mutexes, now that fork() has completed. */
2032 huge_postfork_parent();
2033 base_postfork_parent();
2034 chunk_postfork_parent();
2035 for (i = 0; i < narenas_total; i++) {
2036 if (arenas[i] != NULL)
2037 arena_postfork_parent(arenas[i]);
2038 }
2039 malloc_mutex_postfork_parent(&arenas_lock);
2040 prof_postfork_parent();
2041 ctl_postfork_parent();
2042 }
2043
2044 void
jemalloc_postfork_child(void)2045 jemalloc_postfork_child(void)
2046 {
2047 unsigned i;
2048
2049 assert(malloc_initialized);
2050
2051 /* Release all mutexes, now that fork() has completed. */
2052 huge_postfork_child();
2053 base_postfork_child();
2054 chunk_postfork_child();
2055 for (i = 0; i < narenas_total; i++) {
2056 if (arenas[i] != NULL)
2057 arena_postfork_child(arenas[i]);
2058 }
2059 malloc_mutex_postfork_child(&arenas_lock);
2060 prof_postfork_child();
2061 ctl_postfork_child();
2062 }
2063
2064 void
_malloc_first_thread(void)2065 _malloc_first_thread(void)
2066 {
2067
2068 (void)malloc_mutex_first_thread();
2069 }
2070
2071 /******************************************************************************/
2072 /*
2073 * The following functions are used for TLS allocation/deallocation in static
2074 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc()
2075 * is that these avoid accessing TLS variables.
2076 */
2077
2078 static void *
a0alloc(size_t size,bool zero)2079 a0alloc(size_t size, bool zero)
2080 {
2081
2082 if (malloc_init())
2083 return (NULL);
2084
2085 if (size == 0)
2086 size = 1;
2087
2088 if (size <= arena_maxclass)
2089 return (arena_malloc(arenas[0], size, zero, false));
2090 else
2091 return (huge_malloc(size, zero));
2092 }
2093
2094 void *
a0malloc(size_t size)2095 a0malloc(size_t size)
2096 {
2097
2098 return (a0alloc(size, false));
2099 }
2100
2101 void *
a0calloc(size_t num,size_t size)2102 a0calloc(size_t num, size_t size)
2103 {
2104
2105 return (a0alloc(num * size, true));
2106 }
2107
2108 void
a0free(void * ptr)2109 a0free(void *ptr)
2110 {
2111 arena_chunk_t *chunk;
2112
2113 if (ptr == NULL)
2114 return;
2115
2116 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2117 if (chunk != ptr)
2118 arena_dalloc(chunk->arena, chunk, ptr, false);
2119 else
2120 huge_dalloc(ptr, true);
2121 }
2122
2123 /******************************************************************************/
2124