1 #ifndef JEMALLOC_INTERNAL_H
2 #define JEMALLOC_INTERNAL_H
3
4 #include "jemalloc_internal_defs.h"
5 #include "jemalloc/internal/jemalloc_internal_decls.h"
6
7 #ifdef JEMALLOC_UTRACE
8 #include <sys/ktrace.h>
9 #endif
10
11 #include "un-namespace.h"
12 #include "libc_private.h"
13
14 #define JEMALLOC_NO_DEMANGLE
15 #ifdef JEMALLOC_JET
16 # define JEMALLOC_N(n) jet_##n
17 # include "jemalloc/internal/public_namespace.h"
18 # define JEMALLOC_NO_RENAME
19 # include "../jemalloc.h"
20 # undef JEMALLOC_NO_RENAME
21 #else
22 # define JEMALLOC_N(n) __je_##n
23 # include "../jemalloc.h"
24 #endif
25 #include "jemalloc/internal/private_namespace.h"
26
27 static const bool config_debug =
28 #ifdef JEMALLOC_DEBUG
29 true
30 #else
31 false
32 #endif
33 ;
34 static const bool have_dss =
35 #ifdef JEMALLOC_DSS
36 true
37 #else
38 false
39 #endif
40 ;
41 static const bool config_fill =
42 #ifdef JEMALLOC_FILL
43 true
44 #else
45 false
46 #endif
47 ;
48 static const bool config_lazy_lock = true;
49 static const bool config_prof =
50 #ifdef JEMALLOC_PROF
51 true
52 #else
53 false
54 #endif
55 ;
56 static const bool config_prof_libgcc =
57 #ifdef JEMALLOC_PROF_LIBGCC
58 true
59 #else
60 false
61 #endif
62 ;
63 static const bool config_prof_libunwind =
64 #ifdef JEMALLOC_PROF_LIBUNWIND
65 true
66 #else
67 false
68 #endif
69 ;
70 static const bool maps_coalesce =
71 #ifdef JEMALLOC_MAPS_COALESCE
72 true
73 #else
74 false
75 #endif
76 ;
77 static const bool config_munmap =
78 #ifdef JEMALLOC_MUNMAP
79 true
80 #else
81 false
82 #endif
83 ;
84 static const bool config_stats =
85 #ifdef JEMALLOC_STATS
86 true
87 #else
88 false
89 #endif
90 ;
91 static const bool config_tcache =
92 #ifdef JEMALLOC_TCACHE
93 true
94 #else
95 false
96 #endif
97 ;
98 static const bool config_tls =
99 #ifdef JEMALLOC_TLS
100 true
101 #else
102 false
103 #endif
104 ;
105 static const bool config_utrace =
106 #ifdef JEMALLOC_UTRACE
107 true
108 #else
109 false
110 #endif
111 ;
112 static const bool config_valgrind =
113 #ifdef JEMALLOC_VALGRIND
114 true
115 #else
116 false
117 #endif
118 ;
119 static const bool config_xmalloc =
120 #ifdef JEMALLOC_XMALLOC
121 true
122 #else
123 false
124 #endif
125 ;
126 static const bool config_ivsalloc =
127 #ifdef JEMALLOC_IVSALLOC
128 true
129 #else
130 false
131 #endif
132 ;
133 static const bool config_cache_oblivious =
134 #ifdef JEMALLOC_CACHE_OBLIVIOUS
135 true
136 #else
137 false
138 #endif
139 ;
140
141 #ifdef JEMALLOC_C11ATOMICS
142 #include <stdatomic.h>
143 #endif
144
145 #ifdef JEMALLOC_ATOMIC9
146 #include <machine/atomic.h>
147 #endif
148
149 #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
150 #include <libkern/OSAtomic.h>
151 #endif
152
153 #ifdef JEMALLOC_ZONE
154 #include <mach/mach_error.h>
155 #include <mach/mach_init.h>
156 #include <mach/vm_map.h>
157 #include <malloc/malloc.h>
158 #endif
159
160 #define RB_COMPACT
161 #include "jemalloc/internal/rb.h"
162 #include "jemalloc/internal/qr.h"
163 #include "jemalloc/internal/ql.h"
164
165 /*
166 * jemalloc can conceptually be broken into components (arena, tcache, etc.),
167 * but there are circular dependencies that cannot be broken without
168 * substantial performance degradation. In order to reduce the effect on
169 * visual code flow, read the header files in multiple passes, with one of the
170 * following cpp variables defined during each pass:
171 *
172 * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data
173 * types.
174 * JEMALLOC_H_STRUCTS : Data structures.
175 * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
176 * JEMALLOC_H_INLINES : Inline functions.
177 */
178 /******************************************************************************/
179 #define JEMALLOC_H_TYPES
180
181 #include "jemalloc/internal/jemalloc_internal_macros.h"
182
183 /* Size class index type. */
184 typedef unsigned szind_t;
185
186 /*
187 * Flags bits:
188 *
189 * a: arena
190 * t: tcache
191 * 0: unused
192 * z: zero
193 * n: alignment
194 *
195 * aaaaaaaa aaaatttt tttttttt 0znnnnnn
196 */
197 #define MALLOCX_ARENA_MASK ((int)~0xfffff)
198 #define MALLOCX_ARENA_MAX 0xffe
199 #define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU)
200 #define MALLOCX_TCACHE_MAX 0xffd
201 #define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
202 /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
203 #define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
204 (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
205 #define MALLOCX_ALIGN_GET(flags) \
206 (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
207 #define MALLOCX_ZERO_GET(flags) \
208 ((bool)(flags & MALLOCX_ZERO))
209
210 #define MALLOCX_TCACHE_GET(flags) \
211 (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)
212 #define MALLOCX_ARENA_GET(flags) \
213 (((unsigned)(((unsigned)flags) >> 20)) - 1)
214
215 /* Smallest size class to support. */
216 #define TINY_MIN (1U << LG_TINY_MIN)
217
218 /*
219 * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size
220 * classes).
221 */
222 #ifndef LG_QUANTUM
223 # if (defined(__i386__) || defined(_M_IX86))
224 # define LG_QUANTUM 4
225 # endif
226 # ifdef __ia64__
227 # define LG_QUANTUM 4
228 # endif
229 # ifdef __alpha__
230 # define LG_QUANTUM 4
231 # endif
232 # if (defined(__sparc64__) || defined(__sparcv9))
233 # define LG_QUANTUM 4
234 # endif
235 # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
236 # define LG_QUANTUM 4
237 # endif
238 # ifdef __arm__
239 # define LG_QUANTUM 3
240 # endif
241 # ifdef __aarch64__
242 # define LG_QUANTUM 4
243 # endif
244 # ifdef __hppa__
245 # define LG_QUANTUM 4
246 # endif
247 # ifdef __mips__
248 # define LG_QUANTUM 3
249 # endif
250 # ifdef __or1k__
251 # define LG_QUANTUM 3
252 # endif
253 # ifdef __powerpc__
254 # define LG_QUANTUM 4
255 # endif
256 # ifdef __s390__
257 # define LG_QUANTUM 4
258 # endif
259 # ifdef __SH4__
260 # define LG_QUANTUM 4
261 # endif
262 # ifdef __tile__
263 # define LG_QUANTUM 4
264 # endif
265 # ifdef __le32__
266 # define LG_QUANTUM 4
267 # endif
268 # ifndef LG_QUANTUM
269 # error "Unknown minimum alignment for architecture; specify via "
270 "--with-lg-quantum"
271 # endif
272 #endif
273
274 #define QUANTUM ((size_t)(1U << LG_QUANTUM))
275 #define QUANTUM_MASK (QUANTUM - 1)
276
277 /* Return the smallest quantum multiple that is >= a. */
278 #define QUANTUM_CEILING(a) \
279 (((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
280
281 #define LONG ((size_t)(1U << LG_SIZEOF_LONG))
282 #define LONG_MASK (LONG - 1)
283
284 /* Return the smallest long multiple that is >= a. */
285 #define LONG_CEILING(a) \
286 (((a) + LONG_MASK) & ~LONG_MASK)
287
288 #define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
289 #define PTR_MASK (SIZEOF_PTR - 1)
290
291 /* Return the smallest (void *) multiple that is >= a. */
292 #define PTR_CEILING(a) \
293 (((a) + PTR_MASK) & ~PTR_MASK)
294
295 /*
296 * Maximum size of L1 cache line. This is used to avoid cache line aliasing.
297 * In addition, this controls the spacing of cacheline-spaced size classes.
298 *
299 * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
300 * only handle raw constants.
301 */
302 #define LG_CACHELINE 6
303 #define CACHELINE 64
304 #define CACHELINE_MASK (CACHELINE - 1)
305
306 /* Return the smallest cacheline multiple that is >= s. */
307 #define CACHELINE_CEILING(s) \
308 (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
309
310 /* Page size. LG_PAGE is determined by the configure script. */
311 #ifdef PAGE_MASK
312 # undef PAGE_MASK
313 #endif
314 #define PAGE ((size_t)(1U << LG_PAGE))
315 #define PAGE_MASK ((size_t)(PAGE - 1))
316
317 /* Return the page base address for the page containing address a. */
318 #define PAGE_ADDR2BASE(a) \
319 ((void *)((uintptr_t)(a) & ~PAGE_MASK))
320
321 /* Return the smallest pagesize multiple that is >= s. */
322 #define PAGE_CEILING(s) \
323 (((s) + PAGE_MASK) & ~PAGE_MASK)
324
325 /* Return the nearest aligned address at or below a. */
326 #define ALIGNMENT_ADDR2BASE(a, alignment) \
327 ((void *)((uintptr_t)(a) & (-(alignment))))
328
329 /* Return the offset between a and the nearest aligned address at or below a. */
330 #define ALIGNMENT_ADDR2OFFSET(a, alignment) \
331 ((size_t)((uintptr_t)(a) & (alignment - 1)))
332
333 /* Return the smallest alignment multiple that is >= s. */
334 #define ALIGNMENT_CEILING(s, alignment) \
335 (((s) + (alignment - 1)) & (-(alignment)))
336
337 /* Declare a variable-length array. */
338 #if __STDC_VERSION__ < 199901L
339 # ifdef _MSC_VER
340 # include <malloc.h>
341 # define alloca _alloca
342 # else
343 # ifdef JEMALLOC_HAS_ALLOCA_H
344 # include <alloca.h>
345 # else
346 # include <stdlib.h>
347 # endif
348 # endif
349 # define VARIABLE_ARRAY(type, name, count) \
350 type *name = alloca(sizeof(type) * (count))
351 #else
352 # define VARIABLE_ARRAY(type, name, count) type name[(count)]
353 #endif
354
355 #include "jemalloc/internal/valgrind.h"
356 #include "jemalloc/internal/util.h"
357 #include "jemalloc/internal/atomic.h"
358 #include "jemalloc/internal/prng.h"
359 #include "jemalloc/internal/ckh.h"
360 #include "jemalloc/internal/size_classes.h"
361 #include "jemalloc/internal/stats.h"
362 #include "jemalloc/internal/ctl.h"
363 #include "jemalloc/internal/mutex.h"
364 #include "jemalloc/internal/tsd.h"
365 #include "jemalloc/internal/mb.h"
366 #include "jemalloc/internal/extent.h"
367 #include "jemalloc/internal/arena.h"
368 #include "jemalloc/internal/bitmap.h"
369 #include "jemalloc/internal/base.h"
370 #include "jemalloc/internal/rtree.h"
371 #include "jemalloc/internal/pages.h"
372 #include "jemalloc/internal/chunk.h"
373 #include "jemalloc/internal/huge.h"
374 #include "jemalloc/internal/tcache.h"
375 #include "jemalloc/internal/hash.h"
376 #include "jemalloc/internal/quarantine.h"
377 #include "jemalloc/internal/prof.h"
378
379 #undef JEMALLOC_H_TYPES
380 /******************************************************************************/
381 #define JEMALLOC_H_STRUCTS
382
383 #include "jemalloc/internal/valgrind.h"
384 #include "jemalloc/internal/util.h"
385 #include "jemalloc/internal/atomic.h"
386 #include "jemalloc/internal/prng.h"
387 #include "jemalloc/internal/ckh.h"
388 #include "jemalloc/internal/size_classes.h"
389 #include "jemalloc/internal/stats.h"
390 #include "jemalloc/internal/ctl.h"
391 #include "jemalloc/internal/mutex.h"
392 #include "jemalloc/internal/mb.h"
393 #include "jemalloc/internal/bitmap.h"
394 #define JEMALLOC_ARENA_STRUCTS_A
395 #include "jemalloc/internal/arena.h"
396 #undef JEMALLOC_ARENA_STRUCTS_A
397 #include "jemalloc/internal/extent.h"
398 #define JEMALLOC_ARENA_STRUCTS_B
399 #include "jemalloc/internal/arena.h"
400 #undef JEMALLOC_ARENA_STRUCTS_B
401 #include "jemalloc/internal/base.h"
402 #include "jemalloc/internal/rtree.h"
403 #include "jemalloc/internal/pages.h"
404 #include "jemalloc/internal/chunk.h"
405 #include "jemalloc/internal/huge.h"
406 #include "jemalloc/internal/tcache.h"
407 #include "jemalloc/internal/hash.h"
408 #include "jemalloc/internal/quarantine.h"
409 #include "jemalloc/internal/prof.h"
410
411 #include "jemalloc/internal/tsd.h"
412
413 #undef JEMALLOC_H_STRUCTS
414 /******************************************************************************/
415 #define JEMALLOC_H_EXTERNS
416
417 extern bool opt_abort;
418 extern const char *opt_junk;
419 extern bool opt_junk_alloc;
420 extern bool opt_junk_free;
421 extern size_t opt_quarantine;
422 extern bool opt_redzone;
423 extern bool opt_utrace;
424 extern bool opt_xmalloc;
425 extern bool opt_zero;
426 extern size_t opt_narenas;
427
428 extern bool in_valgrind;
429
430 /* Number of CPUs. */
431 extern unsigned ncpus;
432
433 /*
434 * index2size_tab encodes the same information as could be computed (at
435 * unacceptable cost in some code paths) by index2size_compute().
436 */
437 extern size_t const index2size_tab[NSIZES];
438 /*
439 * size2index_tab is a compact lookup table that rounds request sizes up to
440 * size classes. In order to reduce cache footprint, the table is compressed,
441 * and all accesses are via size2index().
442 */
443 extern uint8_t const size2index_tab[];
444
445 arena_t *a0get(void);
446 void *a0malloc(size_t size);
447 void a0dalloc(void *ptr);
448 void *bootstrap_malloc(size_t size);
449 void *bootstrap_calloc(size_t num, size_t size);
450 void bootstrap_free(void *ptr);
451 arena_t *arenas_extend(unsigned ind);
452 arena_t *arena_init(unsigned ind);
453 unsigned narenas_total_get(void);
454 arena_t *arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing);
455 arena_t *arena_choose_hard(tsd_t *tsd);
456 void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
457 unsigned arena_nbound(unsigned ind);
458 void thread_allocated_cleanup(tsd_t *tsd);
459 void thread_deallocated_cleanup(tsd_t *tsd);
460 void arena_cleanup(tsd_t *tsd);
461 void arenas_cache_cleanup(tsd_t *tsd);
462 void narenas_cache_cleanup(tsd_t *tsd);
463 void arenas_cache_bypass_cleanup(tsd_t *tsd);
464 void jemalloc_prefork(void);
465 void jemalloc_postfork_parent(void);
466 void jemalloc_postfork_child(void);
467
468 #include "jemalloc/internal/valgrind.h"
469 #include "jemalloc/internal/util.h"
470 #include "jemalloc/internal/atomic.h"
471 #include "jemalloc/internal/prng.h"
472 #include "jemalloc/internal/ckh.h"
473 #include "jemalloc/internal/size_classes.h"
474 #include "jemalloc/internal/stats.h"
475 #include "jemalloc/internal/ctl.h"
476 #include "jemalloc/internal/mutex.h"
477 #include "jemalloc/internal/mb.h"
478 #include "jemalloc/internal/bitmap.h"
479 #include "jemalloc/internal/extent.h"
480 #include "jemalloc/internal/arena.h"
481 #include "jemalloc/internal/base.h"
482 #include "jemalloc/internal/rtree.h"
483 #include "jemalloc/internal/pages.h"
484 #include "jemalloc/internal/chunk.h"
485 #include "jemalloc/internal/huge.h"
486 #include "jemalloc/internal/tcache.h"
487 #include "jemalloc/internal/hash.h"
488 #include "jemalloc/internal/quarantine.h"
489 #include "jemalloc/internal/prof.h"
490 #include "jemalloc/internal/tsd.h"
491
492 #undef JEMALLOC_H_EXTERNS
493 /******************************************************************************/
494 #define JEMALLOC_H_INLINES
495
496 #include "jemalloc/internal/valgrind.h"
497 #include "jemalloc/internal/util.h"
498 #include "jemalloc/internal/atomic.h"
499 #include "jemalloc/internal/prng.h"
500 #include "jemalloc/internal/ckh.h"
501 #include "jemalloc/internal/size_classes.h"
502 #include "jemalloc/internal/stats.h"
503 #include "jemalloc/internal/ctl.h"
504 #include "jemalloc/internal/mutex.h"
505 #include "jemalloc/internal/tsd.h"
506 #include "jemalloc/internal/mb.h"
507 #include "jemalloc/internal/extent.h"
508 #include "jemalloc/internal/base.h"
509 #include "jemalloc/internal/rtree.h"
510 #include "jemalloc/internal/pages.h"
511 #include "jemalloc/internal/chunk.h"
512 #include "jemalloc/internal/huge.h"
513
514 #ifndef JEMALLOC_ENABLE_INLINE
515 szind_t size2index_compute(size_t size);
516 szind_t size2index_lookup(size_t size);
517 szind_t size2index(size_t size);
518 size_t index2size_compute(szind_t index);
519 size_t index2size_lookup(szind_t index);
520 size_t index2size(szind_t index);
521 size_t s2u_compute(size_t size);
522 size_t s2u_lookup(size_t size);
523 size_t s2u(size_t size);
524 size_t sa2u(size_t size, size_t alignment);
525 arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
526 arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
527 bool refresh_if_missing);
528 #endif
529
530 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
531 JEMALLOC_INLINE szind_t
size2index_compute(size_t size)532 size2index_compute(size_t size)
533 {
534
535 #if (NTBINS != 0)
536 if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
537 size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
538 size_t lg_ceil = lg_floor(pow2_ceil(size));
539 return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin);
540 }
541 #endif
542 {
543 size_t x = unlikely(ZI(size) < 0) ? ((size<<1) ?
544 (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1))
545 : lg_floor((size<<1)-1);
546 size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 :
547 x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM);
548 size_t grp = shift << LG_SIZE_CLASS_GROUP;
549
550 size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
551 ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
552
553 size_t delta_inverse_mask = ZI(-1) << lg_delta;
554 size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) &
555 ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1);
556
557 size_t index = NTBINS + grp + mod;
558 return (index);
559 }
560 }
561
562 JEMALLOC_ALWAYS_INLINE szind_t
size2index_lookup(size_t size)563 size2index_lookup(size_t size)
564 {
565
566 assert(size <= LOOKUP_MAXCLASS);
567 {
568 size_t ret = ((size_t)(size2index_tab[(size-1) >>
569 LG_TINY_MIN]));
570 assert(ret == size2index_compute(size));
571 return (ret);
572 }
573 }
574
575 JEMALLOC_ALWAYS_INLINE szind_t
size2index(size_t size)576 size2index(size_t size)
577 {
578
579 assert(size > 0);
580 if (likely(size <= LOOKUP_MAXCLASS))
581 return (size2index_lookup(size));
582 return (size2index_compute(size));
583 }
584
585 JEMALLOC_INLINE size_t
index2size_compute(szind_t index)586 index2size_compute(szind_t index)
587 {
588
589 #if (NTBINS > 0)
590 if (index < NTBINS)
591 return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index));
592 #endif
593 {
594 size_t reduced_index = index - NTBINS;
595 size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP;
596 size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) -
597 1);
598
599 size_t grp_size_mask = ~((!!grp)-1);
600 size_t grp_size = ((ZU(1) << (LG_QUANTUM +
601 (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask;
602
603 size_t shift = (grp == 0) ? 1 : grp;
604 size_t lg_delta = shift + (LG_QUANTUM-1);
605 size_t mod_size = (mod+1) << lg_delta;
606
607 size_t usize = grp_size + mod_size;
608 return (usize);
609 }
610 }
611
612 JEMALLOC_ALWAYS_INLINE size_t
index2size_lookup(szind_t index)613 index2size_lookup(szind_t index)
614 {
615 size_t ret = (size_t)index2size_tab[index];
616 assert(ret == index2size_compute(index));
617 return (ret);
618 }
619
620 JEMALLOC_ALWAYS_INLINE size_t
index2size(szind_t index)621 index2size(szind_t index)
622 {
623
624 assert(index < NSIZES);
625 return (index2size_lookup(index));
626 }
627
628 JEMALLOC_ALWAYS_INLINE size_t
s2u_compute(size_t size)629 s2u_compute(size_t size)
630 {
631
632 #if (NTBINS > 0)
633 if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
634 size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
635 size_t lg_ceil = lg_floor(pow2_ceil(size));
636 return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) :
637 (ZU(1) << lg_ceil));
638 }
639 #endif
640 {
641 size_t x = unlikely(ZI(size) < 0) ? ((size<<1) ?
642 (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1))
643 : lg_floor((size<<1)-1);
644 size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
645 ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
646 size_t delta = ZU(1) << lg_delta;
647 size_t delta_mask = delta - 1;
648 size_t usize = (size + delta_mask) & ~delta_mask;
649 return (usize);
650 }
651 }
652
653 JEMALLOC_ALWAYS_INLINE size_t
s2u_lookup(size_t size)654 s2u_lookup(size_t size)
655 {
656 size_t ret = index2size_lookup(size2index_lookup(size));
657
658 assert(ret == s2u_compute(size));
659 return (ret);
660 }
661
662 /*
663 * Compute usable size that would result from allocating an object with the
664 * specified size.
665 */
666 JEMALLOC_ALWAYS_INLINE size_t
s2u(size_t size)667 s2u(size_t size)
668 {
669
670 assert(size > 0);
671 if (likely(size <= LOOKUP_MAXCLASS))
672 return (s2u_lookup(size));
673 return (s2u_compute(size));
674 }
675
676 /*
677 * Compute usable size that would result from allocating an object with the
678 * specified size and alignment.
679 */
680 JEMALLOC_ALWAYS_INLINE size_t
sa2u(size_t size,size_t alignment)681 sa2u(size_t size, size_t alignment)
682 {
683 size_t usize;
684
685 assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
686
687 /* Try for a small size class. */
688 if (size <= SMALL_MAXCLASS && alignment < PAGE) {
689 /*
690 * Round size up to the nearest multiple of alignment.
691 *
692 * This done, we can take advantage of the fact that for each
693 * small size class, every object is aligned at the smallest
694 * power of two that is non-zero in the base two representation
695 * of the size. For example:
696 *
697 * Size | Base 2 | Minimum alignment
698 * -----+----------+------------------
699 * 96 | 1100000 | 32
700 * 144 | 10100000 | 32
701 * 192 | 11000000 | 64
702 */
703 usize = s2u(ALIGNMENT_CEILING(size, alignment));
704 if (usize < LARGE_MINCLASS)
705 return (usize);
706 }
707
708 /* Try for a large size class. */
709 if (likely(size <= large_maxclass) && likely(alignment < chunksize)) {
710 /*
711 * We can't achieve subpage alignment, so round up alignment
712 * to the minimum that can actually be supported.
713 */
714 alignment = PAGE_CEILING(alignment);
715
716 /* Make sure result is a large size class. */
717 usize = (size <= LARGE_MINCLASS) ? LARGE_MINCLASS : s2u(size);
718
719 /*
720 * Calculate the size of the over-size run that arena_palloc()
721 * would need to allocate in order to guarantee the alignment.
722 */
723 if (usize + large_pad + alignment - PAGE <= arena_maxrun)
724 return (usize);
725 }
726
727 /* Huge size class. Beware of size_t overflow. */
728
729 /*
730 * We can't achieve subchunk alignment, so round up alignment to the
731 * minimum that can actually be supported.
732 */
733 alignment = CHUNK_CEILING(alignment);
734 if (alignment == 0) {
735 /* size_t overflow. */
736 return (0);
737 }
738
739 /* Make sure result is a huge size class. */
740 if (size <= chunksize)
741 usize = chunksize;
742 else {
743 usize = s2u(size);
744 if (usize < size) {
745 /* size_t overflow. */
746 return (0);
747 }
748 }
749
750 /*
751 * Calculate the multi-chunk mapping that huge_palloc() would need in
752 * order to guarantee the alignment.
753 */
754 if (usize + alignment - PAGE < usize) {
755 /* size_t overflow. */
756 return (0);
757 }
758 return (usize);
759 }
760
761 /* Choose an arena based on a per-thread value. */
762 JEMALLOC_INLINE arena_t *
arena_choose(tsd_t * tsd,arena_t * arena)763 arena_choose(tsd_t *tsd, arena_t *arena)
764 {
765 arena_t *ret;
766
767 if (arena != NULL)
768 return (arena);
769
770 if (unlikely((ret = tsd_arena_get(tsd)) == NULL))
771 ret = arena_choose_hard(tsd);
772
773 return (ret);
774 }
775
776 JEMALLOC_INLINE arena_t *
arena_get(tsd_t * tsd,unsigned ind,bool init_if_missing,bool refresh_if_missing)777 arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
778 bool refresh_if_missing)
779 {
780 arena_t *arena;
781 arena_t **arenas_cache = tsd_arenas_cache_get(tsd);
782
783 /* init_if_missing requires refresh_if_missing. */
784 assert(!init_if_missing || refresh_if_missing);
785
786 if (unlikely(arenas_cache == NULL)) {
787 /* arenas_cache hasn't been initialized yet. */
788 return (arena_get_hard(tsd, ind, init_if_missing));
789 }
790 if (unlikely(ind >= tsd_narenas_cache_get(tsd))) {
791 /*
792 * ind is invalid, cache is old (too small), or arena to be
793 * initialized.
794 */
795 return (refresh_if_missing ? arena_get_hard(tsd, ind,
796 init_if_missing) : NULL);
797 }
798 arena = arenas_cache[ind];
799 if (likely(arena != NULL) || !refresh_if_missing)
800 return (arena);
801 return (arena_get_hard(tsd, ind, init_if_missing));
802 }
803 #endif
804
805 #include "jemalloc/internal/bitmap.h"
806 /*
807 * Include portions of arena.h interleaved with tcache.h in order to resolve
808 * circular dependencies.
809 */
810 #define JEMALLOC_ARENA_INLINE_A
811 #include "jemalloc/internal/arena.h"
812 #undef JEMALLOC_ARENA_INLINE_A
813 #include "jemalloc/internal/tcache.h"
814 #define JEMALLOC_ARENA_INLINE_B
815 #include "jemalloc/internal/arena.h"
816 #undef JEMALLOC_ARENA_INLINE_B
817 #include "jemalloc/internal/hash.h"
818 #include "jemalloc/internal/quarantine.h"
819
820 #ifndef JEMALLOC_ENABLE_INLINE
821 arena_t *iaalloc(const void *ptr);
822 size_t isalloc(const void *ptr, bool demote);
823 void *iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache,
824 bool is_metadata, arena_t *arena);
825 void *imalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena);
826 void *imalloc(tsd_t *tsd, size_t size);
827 void *icalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena);
828 void *icalloc(tsd_t *tsd, size_t size);
829 void *ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
830 tcache_t *tcache, bool is_metadata, arena_t *arena);
831 void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
832 tcache_t *tcache, arena_t *arena);
833 void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
834 size_t ivsalloc(const void *ptr, bool demote);
835 size_t u2rz(size_t usize);
836 size_t p2rz(const void *ptr);
837 void idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata);
838 void idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache);
839 void idalloc(tsd_t *tsd, void *ptr);
840 void iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache);
841 void isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache);
842 void isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache);
843 void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
844 size_t extra, size_t alignment, bool zero, tcache_t *tcache,
845 arena_t *arena);
846 void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
847 size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
848 void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
849 size_t alignment, bool zero);
850 bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra,
851 size_t alignment, bool zero);
852 #endif
853
854 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
855 JEMALLOC_ALWAYS_INLINE arena_t *
iaalloc(const void * ptr)856 iaalloc(const void *ptr)
857 {
858
859 assert(ptr != NULL);
860
861 return (arena_aalloc(ptr));
862 }
863
864 /*
865 * Typical usage:
866 * void *ptr = [...]
867 * size_t sz = isalloc(ptr, config_prof);
868 */
869 JEMALLOC_ALWAYS_INLINE size_t
isalloc(const void * ptr,bool demote)870 isalloc(const void *ptr, bool demote)
871 {
872
873 assert(ptr != NULL);
874 /* Demotion only makes sense if config_prof is true. */
875 assert(config_prof || !demote);
876
877 return (arena_salloc(ptr, demote));
878 }
879
880 JEMALLOC_ALWAYS_INLINE void *
iallocztm(tsd_t * tsd,size_t size,bool zero,tcache_t * tcache,bool is_metadata,arena_t * arena)881 iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache, bool is_metadata,
882 arena_t *arena)
883 {
884 void *ret;
885
886 assert(size != 0);
887
888 ret = arena_malloc(tsd, arena, size, zero, tcache);
889 if (config_stats && is_metadata && likely(ret != NULL)) {
890 arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
891 config_prof));
892 }
893 return (ret);
894 }
895
896 JEMALLOC_ALWAYS_INLINE void *
imalloct(tsd_t * tsd,size_t size,tcache_t * tcache,arena_t * arena)897 imalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena)
898 {
899
900 return (iallocztm(tsd, size, false, tcache, false, arena));
901 }
902
903 JEMALLOC_ALWAYS_INLINE void *
imalloc(tsd_t * tsd,size_t size)904 imalloc(tsd_t *tsd, size_t size)
905 {
906
907 return (iallocztm(tsd, size, false, tcache_get(tsd, true), false, NULL));
908 }
909
910 JEMALLOC_ALWAYS_INLINE void *
icalloct(tsd_t * tsd,size_t size,tcache_t * tcache,arena_t * arena)911 icalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena)
912 {
913
914 return (iallocztm(tsd, size, true, tcache, false, arena));
915 }
916
917 JEMALLOC_ALWAYS_INLINE void *
icalloc(tsd_t * tsd,size_t size)918 icalloc(tsd_t *tsd, size_t size)
919 {
920
921 return (iallocztm(tsd, size, true, tcache_get(tsd, true), false, NULL));
922 }
923
924 JEMALLOC_ALWAYS_INLINE void *
ipallocztm(tsd_t * tsd,size_t usize,size_t alignment,bool zero,tcache_t * tcache,bool is_metadata,arena_t * arena)925 ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
926 tcache_t *tcache, bool is_metadata, arena_t *arena)
927 {
928 void *ret;
929
930 assert(usize != 0);
931 assert(usize == sa2u(usize, alignment));
932
933 ret = arena_palloc(tsd, arena, usize, alignment, zero, tcache);
934 assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
935 if (config_stats && is_metadata && likely(ret != NULL)) {
936 arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
937 config_prof));
938 }
939 return (ret);
940 }
941
942 JEMALLOC_ALWAYS_INLINE void *
ipalloct(tsd_t * tsd,size_t usize,size_t alignment,bool zero,tcache_t * tcache,arena_t * arena)943 ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
944 tcache_t *tcache, arena_t *arena)
945 {
946
947 return (ipallocztm(tsd, usize, alignment, zero, tcache, false, arena));
948 }
949
950 JEMALLOC_ALWAYS_INLINE void *
ipalloc(tsd_t * tsd,size_t usize,size_t alignment,bool zero)951 ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
952 {
953
954 return (ipallocztm(tsd, usize, alignment, zero, tcache_get(tsd,
955 NULL), false, NULL));
956 }
957
958 JEMALLOC_ALWAYS_INLINE size_t
ivsalloc(const void * ptr,bool demote)959 ivsalloc(const void *ptr, bool demote)
960 {
961 extent_node_t *node;
962
963 /* Return 0 if ptr is not within a chunk managed by jemalloc. */
964 node = chunk_lookup(ptr, false);
965 if (node == NULL)
966 return (0);
967 /* Only arena chunks should be looked up via interior pointers. */
968 assert(extent_node_addr_get(node) == ptr ||
969 extent_node_achunk_get(node));
970
971 return (isalloc(ptr, demote));
972 }
973
974 JEMALLOC_INLINE size_t
u2rz(size_t usize)975 u2rz(size_t usize)
976 {
977 size_t ret;
978
979 if (usize <= SMALL_MAXCLASS) {
980 szind_t binind = size2index(usize);
981 ret = arena_bin_info[binind].redzone_size;
982 } else
983 ret = 0;
984
985 return (ret);
986 }
987
988 JEMALLOC_INLINE size_t
p2rz(const void * ptr)989 p2rz(const void *ptr)
990 {
991 size_t usize = isalloc(ptr, false);
992
993 return (u2rz(usize));
994 }
995
996 JEMALLOC_ALWAYS_INLINE void
idalloctm(tsd_t * tsd,void * ptr,tcache_t * tcache,bool is_metadata)997 idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata)
998 {
999
1000 assert(ptr != NULL);
1001 if (config_stats && is_metadata) {
1002 arena_metadata_allocated_sub(iaalloc(ptr), isalloc(ptr,
1003 config_prof));
1004 }
1005
1006 arena_dalloc(tsd, ptr, tcache);
1007 }
1008
1009 JEMALLOC_ALWAYS_INLINE void
idalloct(tsd_t * tsd,void * ptr,tcache_t * tcache)1010 idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache)
1011 {
1012
1013 idalloctm(tsd, ptr, tcache, false);
1014 }
1015
1016 JEMALLOC_ALWAYS_INLINE void
idalloc(tsd_t * tsd,void * ptr)1017 idalloc(tsd_t *tsd, void *ptr)
1018 {
1019
1020 idalloctm(tsd, ptr, tcache_get(tsd, false), false);
1021 }
1022
1023 JEMALLOC_ALWAYS_INLINE void
iqalloc(tsd_t * tsd,void * ptr,tcache_t * tcache)1024 iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
1025 {
1026
1027 if (config_fill && unlikely(opt_quarantine))
1028 quarantine(tsd, ptr);
1029 else
1030 idalloctm(tsd, ptr, tcache, false);
1031 }
1032
1033 JEMALLOC_ALWAYS_INLINE void
isdalloct(tsd_t * tsd,void * ptr,size_t size,tcache_t * tcache)1034 isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
1035 {
1036
1037 arena_sdalloc(tsd, ptr, size, tcache);
1038 }
1039
1040 JEMALLOC_ALWAYS_INLINE void
isqalloc(tsd_t * tsd,void * ptr,size_t size,tcache_t * tcache)1041 isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
1042 {
1043
1044 if (config_fill && unlikely(opt_quarantine))
1045 quarantine(tsd, ptr);
1046 else
1047 isdalloct(tsd, ptr, size, tcache);
1048 }
1049
1050 JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsd_t * tsd,void * ptr,size_t oldsize,size_t size,size_t extra,size_t alignment,bool zero,tcache_t * tcache,arena_t * arena)1051 iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
1052 size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
1053 {
1054 void *p;
1055 size_t usize, copysize;
1056
1057 usize = sa2u(size + extra, alignment);
1058 if (usize == 0)
1059 return (NULL);
1060 p = ipalloct(tsd, usize, alignment, zero, tcache, arena);
1061 if (p == NULL) {
1062 if (extra == 0)
1063 return (NULL);
1064 /* Try again, without extra this time. */
1065 usize = sa2u(size, alignment);
1066 if (usize == 0)
1067 return (NULL);
1068 p = ipalloct(tsd, usize, alignment, zero, tcache, arena);
1069 if (p == NULL)
1070 return (NULL);
1071 }
1072 /*
1073 * Copy at most size bytes (not size+extra), since the caller has no
1074 * expectation that the extra bytes will be reliably preserved.
1075 */
1076 copysize = (size < oldsize) ? size : oldsize;
1077 memcpy(p, ptr, copysize);
1078 isqalloc(tsd, ptr, oldsize, tcache);
1079 return (p);
1080 }
1081
1082 JEMALLOC_ALWAYS_INLINE void *
iralloct(tsd_t * tsd,void * ptr,size_t oldsize,size_t size,size_t alignment,bool zero,tcache_t * tcache,arena_t * arena)1083 iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
1084 bool zero, tcache_t *tcache, arena_t *arena)
1085 {
1086
1087 assert(ptr != NULL);
1088 assert(size != 0);
1089
1090 if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
1091 != 0) {
1092 /*
1093 * Existing object alignment is inadequate; allocate new space
1094 * and copy.
1095 */
1096 return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
1097 zero, tcache, arena));
1098 }
1099
1100 return (arena_ralloc(tsd, arena, ptr, oldsize, size, alignment, zero,
1101 tcache));
1102 }
1103
1104 JEMALLOC_ALWAYS_INLINE void *
iralloc(tsd_t * tsd,void * ptr,size_t oldsize,size_t size,size_t alignment,bool zero)1105 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
1106 bool zero)
1107 {
1108
1109 return (iralloct(tsd, ptr, oldsize, size, alignment, zero,
1110 tcache_get(tsd, true), NULL));
1111 }
1112
1113 JEMALLOC_ALWAYS_INLINE bool
ixalloc(void * ptr,size_t oldsize,size_t size,size_t extra,size_t alignment,bool zero)1114 ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment,
1115 bool zero)
1116 {
1117
1118 assert(ptr != NULL);
1119 assert(size != 0);
1120
1121 if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
1122 != 0) {
1123 /* Existing object alignment is inadequate. */
1124 return (true);
1125 }
1126
1127 return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
1128 }
1129 #endif
1130
1131 #include "jemalloc/internal/prof.h"
1132
1133 #undef JEMALLOC_H_INLINES
1134 /******************************************************************************/
1135 #endif /* JEMALLOC_INTERNAL_H */
1136