1 /* 2 * Copyright (c) 2019-2021 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 20 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 26 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #ifndef _SYS__MALLOC_H_ 31 #define _SYS__MALLOC_H_ 32 33 /* 34 * Do not include this header outside _KERNEL or _KERNEL_STRUCTURES scopes. 35 * Used in <sys/user.h>. 36 */ 37 38 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES) 39 #include <sys/cdefs.h> /* for __cache_align */ 40 #include <sys/spinlock.h> /* for spinlock */ 41 #include <sys/exislock.h> /* for exislock_t */ 42 #include <machine/stdint.h> /* for __* types */ 43 #include <machine/param.h> /* for SMP_MAXCPU */ 44 45 /* 46 * A kmalloc slab (used with KSF_OBJSIZE) holds N fixed-size objects 47 * in a fixed (typically 32KB) block of memory prefaced by the structure. 48 */ 49 #define KMALLOC_SLAB_SIZE (__size_t)(128 * 1024) 50 #define KMALLOC_SLAB_MASK ((__size_t)(KMALLOC_SLAB_SIZE - 1)) 51 52 #define KMALLOC_SLAB_MAXOBJS (KMALLOC_SLAB_SIZE / __VM_CACHELINE_SIZE) 53 #define KMALLOC_LOOSE_SIZE (KMALLOC_SLAB_SIZE * 4) 54 55 #define KMALLOC_SLAB_MAGIC 0x6b736c62 56 #define KMALLOC_MAXFREEMAGS 4 57 58 #define KMALLOC_CHECK_DOUBLE_FREE 59 60 struct kmalloc_slab { 61 struct spinlock spin; 62 struct kmalloc_slab *next; /* next mag in list */ 63 struct malloc_type *type; /* who does this belong to */ 64 __uint32_t magic; 65 __uint32_t orig_cpuid; /* originally allocated on */ 66 __size_t offset; /* copied from kmalloc_mgt */ 67 __size_t objsize; /* copied from malloc_type */ 68 __size_t ncount; 69 /* copy of slab_count from kmalloc_mgt */ 70 __size_t aindex; /* start of allocations */ 71 __size_t findex; /* end of frees */ 72 __size_t xindex; /* synchronizer */ 73 exislock_t exis; /* existential lock state */ 74 void *unused01; 75 __uint64_t bmap[(KMALLOC_SLAB_MAXOBJS + 63) / 64]; 76 void *fobjs[1]; 77 /* 78 * Circular buffer listing free objects. All indices modulo ncount. 79 * Invariants: 80 * - aindex, findex and xindex never decrease 81 * - aindex <= xindex <= findex 82 * 83 * aindex <= i < xindex: fobjs[i % count] is a pointer to a free object. 84 * xindex <= i < findex: 85 * Synchronization state: fobjs[i % count] is temporarily reserved. 86 */ 87 } __cachealign; 88 89 /* 90 * pcpu slab management structure for kmalloc zone. 91 * 92 * The intent is to try to improve cache characteristics and to reduce 93 * fragmentation by keeping collections localized. See the code for 94 * more information. 95 * 96 * 'active' and 'alternate' are single slabs while 'partial', 'full', and 97 * 'empty' are lists of slabs. 98 * 99 * The kmalloc zone also uses one of these as a global management structure 100 * excess emptymags are regularly moved to the global structure. This mgt 101 * structure is used GLOBALLY (per-zone) as well as PCPU (per-zone). The 102 * fields are utilized differently (or not at all), depending on the use case. 103 */ 104 struct kmalloc_mgt { 105 struct spinlock spin; 106 struct kmalloc_slab *active; /* pcpu */ 107 struct kmalloc_slab *alternate; /* pcpu */ 108 struct kmalloc_slab *partial; /* global */ 109 struct kmalloc_slab *full; /* global */ 110 struct kmalloc_slab *empty; /* global */ 111 struct kmalloc_slab **empty_tailp; /* global */ 112 __size_t slab_offset; /* first object in slab */ 113 __size_t slab_count; /* objects per slab */ 114 __size_t npartial; /* counts */ 115 __size_t nfull; 116 __size_t nempty; 117 __size_t gcache_count; /* #slabs returned to gcache */ 118 __size_t unused01; 119 __size_t unused02; 120 } __cachealign; 121 122 /* 123 * The malloc tracking structure. Note that per-cpu entries must be 124 * aggregated for accurate statistics, they do not actually break the 125 * stats down by cpu (e.g. the cpu freeing memory will subtract from 126 * its slot, not the originating cpu's slot). 127 * 128 * SMP_MAXCPU is used so modules which use malloc remain compatible 129 * between UP and SMP. 130 * 131 * WARNING: __cachealign typically represents 64 byte alignment, so 132 * this structure may be larger than expected. 133 * 134 * WARNING: loosememuse is transfered to ks_loosememuse and zerod 135 * often (e.g. uses atomic_swap_long()). It allows pcpu 136 * updates to be taken into account without causing lots 137 * of cache ping-pongs 138 */ 139 struct kmalloc_use { 140 __size_t memuse; 141 __size_t inuse; 142 __int64_t calls; /* allocations counter (total) */ 143 __size_t loosememuse; 144 struct kmalloc_mgt mgt; /* pcpu object store */ 145 } __cachealign; 146 147 struct malloc_type { 148 struct malloc_type *ks_next; /* next in list */ 149 __size_t ks_loosememuse; /* (inaccurate) aggregate memuse */ 150 __size_t ks_limit; /* most that are allowed to exist */ 151 __uint64_t ks_unused0; 152 __uint32_t ks_flags; /* KSF_x flags */ 153 __uint32_t ks_magic; /* if it's not magic, don't touch it */ 154 const char *ks_shortdesc; /* short description */ 155 __size_t ks_objsize; /* single size if non-zero */ 156 struct kmalloc_use *ks_use; 157 struct kmalloc_use ks_use0; /* dummy prior to SMP startup */ 158 struct kmalloc_mgt ks_mgt; /* rollup object store */ 159 }; 160 161 typedef struct malloc_type *malloc_type_t; 162 163 #define MALLOC_DECLARE(type) \ 164 extern struct malloc_type type[1] /* ref as ptr */ 165 166 #define KSF_OBJSIZE 0x00000001 /* zone used for one object type/size */ 167 #define KSF_POLLING 0x00000002 /* poll in progress */ 168 169 #define KMGD_MAXFREESLABS 128 170 171 typedef struct KMGlobalData { 172 struct kmalloc_slab *free_slabs; 173 struct kmalloc_slab *remote_free_slabs; 174 __size_t free_count; 175 void *reserved[5]; 176 } KMGlobalData; 177 178 #endif 179 180 #endif /* !_SYS__MALLOC_H_ */ 181