1 /*-
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * uma_core.c Implementation of the Universal Memory allocator
31 *
32 * This allocator is intended to replace the multitude of similar object caches
33 * in the standard FreeBSD kernel. The intent is to be flexible as well as
34 * effecient. A primary design goal is to return unused memory to the rest of
35 * the system. This will make the system as a whole more flexible due to the
36 * ability to move memory to subsystems which most need it instead of leaving
37 * pools of reserved memory unused.
38 *
39 * The basic ideas stem from similar slab/zone based allocators whose algorithms
40 * are well known.
41 *
42 */
43
44 /*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50 #include <sys/cdefs.h>
51 __FBSDID("$FreeBSD$");
52
53 /* I should really use ktr.. */
54 /*
55 #define UMA_DEBUG 1
56 #define UMA_DEBUG_ALLOC 1
57 #define UMA_DEBUG_ALLOC_1 1
58 */
59
60 #include "opt_ddb.h"
61 #include "opt_param.h"
62 #include "opt_vm.h"
63
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/bitset.h>
67 #include <sys/kernel.h>
68 #include <sys/types.h>
69 #include <sys/queue.h>
70 #include <sys/malloc.h>
71 #include <sys/ktr.h>
72 #include <sys/lock.h>
73 #include <sys/sysctl.h>
74 #include <sys/mutex.h>
75 #include <sys/proc.h>
76 #include <sys/rwlock.h>
77 #include <sys/sbuf.h>
78 #include <sys/sched.h>
79 #include <sys/smp.h>
80 #include <sys/vmmeter.h>
81
82 #include <vm/vm.h>
83 #include <vm/vm_object.h>
84 #include <vm/vm_page.h>
85 #include <vm/vm_pageout.h>
86 #include <vm/vm_param.h>
87 #include <vm/vm_map.h>
88 #include <vm/vm_kern.h>
89 #include <vm/vm_extern.h>
90 #include <vm/uma.h>
91 #include <vm/uma_int.h>
92 #include <vm/uma_dbg.h>
93
94 #include <ddb/ddb.h>
95
96 #ifdef DEBUG_MEMGUARD
97 #include <vm/memguard.h>
98 #endif
99
100 /*
101 * This is the zone and keg from which all zones are spawned. The idea is that
102 * even the zone & keg heads are allocated from the allocator, so we use the
103 * bss section to bootstrap us.
104 */
105 static struct uma_keg masterkeg;
106 static struct uma_zone masterzone_k;
107 static struct uma_zone masterzone_z;
108 static uma_zone_t kegs = &masterzone_k;
109 static uma_zone_t zones = &masterzone_z;
110
111 /* This is the zone from which all of uma_slab_t's are allocated. */
112 static uma_zone_t slabzone;
113 static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */
114
115 /*
116 * The initial hash tables come out of this zone so they can be allocated
117 * prior to malloc coming up.
118 */
119 static uma_zone_t hashzone;
120
121 /* The boot-time adjusted value for cache line alignment. */
122 int uma_align_cache = 64 - 1;
123
124 static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
125
126 /*
127 * Are we allowed to allocate buckets?
128 */
129 static int bucketdisable = 1;
130
131 /* Linked list of all kegs in the system */
132 static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
133
134 /* Linked list of all cache-only zones in the system */
135 static LIST_HEAD(,uma_zone) uma_cachezones =
136 LIST_HEAD_INITIALIZER(uma_cachezones);
137
138 /* This RW lock protects the keg list */
139 static struct rwlock_padalign uma_rwlock;
140
141 /* Linked list of boot time pages */
142 static LIST_HEAD(,uma_slab) uma_boot_pages =
143 LIST_HEAD_INITIALIZER(uma_boot_pages);
144
145 /* This mutex protects the boot time pages list */
146 static struct mtx_padalign uma_boot_pages_mtx;
147
148 static struct sx uma_drain_lock;
149
150 /* Is the VM done starting up? */
151 static int booted = 0;
152 #define UMA_STARTUP 1
153 #define UMA_STARTUP2 2
154
155 /*
156 * Only mbuf clusters use ref zones. Just provide enough references
157 * to support the one user. New code should not use the ref facility.
158 */
159 static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
160
161 /*
162 * This is the handle used to schedule events that need to happen
163 * outside of the allocation fast path.
164 */
165 static struct callout uma_callout;
166 #define UMA_TIMEOUT 20 /* Seconds for callout interval. */
167
168 /*
169 * This structure is passed as the zone ctor arg so that I don't have to create
170 * a special allocation function just for zones.
171 */
172 struct uma_zctor_args {
173 const char *name;
174 size_t size;
175 uma_ctor ctor;
176 uma_dtor dtor;
177 uma_init uminit;
178 uma_fini fini;
179 uma_import import;
180 uma_release release;
181 void *arg;
182 uma_keg_t keg;
183 int align;
184 uint32_t flags;
185 };
186
187 struct uma_kctor_args {
188 uma_zone_t zone;
189 size_t size;
190 uma_init uminit;
191 uma_fini fini;
192 int align;
193 uint32_t flags;
194 };
195
196 struct uma_bucket_zone {
197 uma_zone_t ubz_zone;
198 char *ubz_name;
199 int ubz_entries; /* Number of items it can hold. */
200 int ubz_maxsize; /* Maximum allocation size per-item. */
201 };
202
203 /*
204 * Compute the actual number of bucket entries to pack them in power
205 * of two sizes for more efficient space utilization.
206 */
207 #define BUCKET_SIZE(n) \
208 (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *))
209
210 #define BUCKET_MAX BUCKET_SIZE(256)
211
212 struct uma_bucket_zone bucket_zones[] = {
213 { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
214 { NULL, "6 Bucket", BUCKET_SIZE(6), 3072 },
215 { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
216 { NULL, "12 Bucket", BUCKET_SIZE(12), 1536 },
217 { NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
218 { NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
219 { NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
220 { NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
221 { NULL, "256 Bucket", BUCKET_SIZE(256), 64 },
222 { NULL, NULL, 0}
223 };
224
225 /*
226 * Flags and enumerations to be passed to internal functions.
227 */
228 enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
229
230 /* Prototypes.. */
231
232 static void *noobj_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
233 static void *page_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
234 static void *startup_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
235 static void page_free(void *, vm_size_t, uint8_t);
236 static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
237 static void cache_drain(uma_zone_t);
238 static void bucket_drain(uma_zone_t, uma_bucket_t);
239 static void bucket_cache_drain(uma_zone_t zone);
240 static int keg_ctor(void *, int, void *, int);
241 static void keg_dtor(void *, int, void *);
242 static int zone_ctor(void *, int, void *, int);
243 static void zone_dtor(void *, int, void *);
244 static int zero_init(void *, int, int);
245 static void keg_small_init(uma_keg_t keg);
246 static void keg_large_init(uma_keg_t keg);
247 static void zone_foreach(void (*zfunc)(uma_zone_t));
248 static void zone_timeout(uma_zone_t zone);
249 static int hash_alloc(struct uma_hash *);
250 static int hash_expand(struct uma_hash *, struct uma_hash *);
251 static void hash_free(struct uma_hash *hash);
252 static void uma_timeout(void *);
253 static void uma_startup3(void);
254 static void *zone_alloc_item(uma_zone_t, void *, int);
255 static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
256 static void bucket_enable(void);
257 static void bucket_init(void);
258 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
259 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
260 static void bucket_zone_drain(void);
261 static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
262 static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
263 static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
264 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
265 static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
266 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
267 uma_fini fini, int align, uint32_t flags);
268 static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
269 static void zone_release(uma_zone_t zone, void **bucket, int cnt);
270 static void uma_zero_item(void *item, uma_zone_t zone);
271
272 void uma_print_zone(uma_zone_t);
273 void uma_print_stats(void);
274 static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
275 static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
276
277 SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
278
279 SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
280 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
281
282 SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
283 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
284
285 static int zone_warnings = 1;
286 SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0,
287 "Warn when UMA zones becomes full");
288
289 /*
290 * This routine checks to see whether or not it's safe to enable buckets.
291 */
292 static void
bucket_enable(void)293 bucket_enable(void)
294 {
295 bucketdisable = vm_page_count_min();
296 }
297
298 /*
299 * Initialize bucket_zones, the array of zones of buckets of various sizes.
300 *
301 * For each zone, calculate the memory required for each bucket, consisting
302 * of the header and an array of pointers.
303 */
304 static void
bucket_init(void)305 bucket_init(void)
306 {
307 struct uma_bucket_zone *ubz;
308 int size;
309
310 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) {
311 size = roundup(sizeof(struct uma_bucket), sizeof(void *));
312 size += sizeof(void *) * ubz->ubz_entries;
313 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
314 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
315 UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
316 }
317 }
318
319 /*
320 * Given a desired number of entries for a bucket, return the zone from which
321 * to allocate the bucket.
322 */
323 static struct uma_bucket_zone *
bucket_zone_lookup(int entries)324 bucket_zone_lookup(int entries)
325 {
326 struct uma_bucket_zone *ubz;
327
328 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
329 if (ubz->ubz_entries >= entries)
330 return (ubz);
331 ubz--;
332 return (ubz);
333 }
334
335 static int
bucket_select(int size)336 bucket_select(int size)
337 {
338 struct uma_bucket_zone *ubz;
339
340 ubz = &bucket_zones[0];
341 if (size > ubz->ubz_maxsize)
342 return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1);
343
344 for (; ubz->ubz_entries != 0; ubz++)
345 if (ubz->ubz_maxsize < size)
346 break;
347 ubz--;
348 return (ubz->ubz_entries);
349 }
350
351 static uma_bucket_t
bucket_alloc(uma_zone_t zone,void * udata,int flags)352 bucket_alloc(uma_zone_t zone, void *udata, int flags)
353 {
354 struct uma_bucket_zone *ubz;
355 uma_bucket_t bucket;
356
357 /*
358 * This is to stop us from allocating per cpu buckets while we're
359 * running out of vm.boot_pages. Otherwise, we would exhaust the
360 * boot pages. This also prevents us from allocating buckets in
361 * low memory situations.
362 */
363 if (bucketdisable)
364 return (NULL);
365 /*
366 * To limit bucket recursion we store the original zone flags
367 * in a cookie passed via zalloc_arg/zfree_arg. This allows the
368 * NOVM flag to persist even through deep recursions. We also
369 * store ZFLAG_BUCKET once we have recursed attempting to allocate
370 * a bucket for a bucket zone so we do not allow infinite bucket
371 * recursion. This cookie will even persist to frees of unused
372 * buckets via the allocation path or bucket allocations in the
373 * free path.
374 */
375 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
376 udata = (void *)(uintptr_t)zone->uz_flags;
377 else {
378 if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
379 return (NULL);
380 udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
381 }
382 if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
383 flags |= M_NOVM;
384 ubz = bucket_zone_lookup(zone->uz_count);
385 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
386 ubz++;
387 bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
388 if (bucket) {
389 #ifdef INVARIANTS
390 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
391 #endif
392 bucket->ub_cnt = 0;
393 bucket->ub_entries = ubz->ubz_entries;
394 }
395
396 return (bucket);
397 }
398
399 static void
bucket_free(uma_zone_t zone,uma_bucket_t bucket,void * udata)400 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
401 {
402 struct uma_bucket_zone *ubz;
403
404 KASSERT(bucket->ub_cnt == 0,
405 ("bucket_free: Freeing a non free bucket."));
406 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
407 udata = (void *)(uintptr_t)zone->uz_flags;
408 ubz = bucket_zone_lookup(bucket->ub_entries);
409 uma_zfree_arg(ubz->ubz_zone, bucket, udata);
410 }
411
412 static void
bucket_zone_drain(void)413 bucket_zone_drain(void)
414 {
415 struct uma_bucket_zone *ubz;
416
417 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
418 zone_drain(ubz->ubz_zone);
419 }
420
421 static void
zone_log_warning(uma_zone_t zone)422 zone_log_warning(uma_zone_t zone)
423 {
424 static const struct timeval warninterval = { 300, 0 };
425
426 if (!zone_warnings || zone->uz_warning == NULL)
427 return;
428
429 if (ratecheck(&zone->uz_ratecheck, &warninterval))
430 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
431 }
432
433 static inline void
zone_maxaction(uma_zone_t zone)434 zone_maxaction(uma_zone_t zone)
435 {
436 if (zone->uz_maxaction)
437 (*zone->uz_maxaction)(zone);
438 }
439
440 static void
zone_foreach_keg(uma_zone_t zone,void (* kegfn)(uma_keg_t))441 zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
442 {
443 uma_klink_t klink;
444
445 LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
446 kegfn(klink->kl_keg);
447 }
448
449 /*
450 * Routine called by timeout which is used to fire off some time interval
451 * based calculations. (stats, hash size, etc.)
452 *
453 * Arguments:
454 * arg Unused
455 *
456 * Returns:
457 * Nothing
458 */
459 static void
uma_timeout(void * unused)460 uma_timeout(void *unused)
461 {
462 bucket_enable();
463 zone_foreach(zone_timeout);
464
465 /* Reschedule this event */
466 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
467 }
468
469 /*
470 * Routine to perform timeout driven calculations. This expands the
471 * hashes and does per cpu statistics aggregation.
472 *
473 * Returns nothing.
474 */
475 static void
keg_timeout(uma_keg_t keg)476 keg_timeout(uma_keg_t keg)
477 {
478
479 KEG_LOCK(keg);
480 /*
481 * Expand the keg hash table.
482 *
483 * This is done if the number of slabs is larger than the hash size.
484 * What I'm trying to do here is completely reduce collisions. This
485 * may be a little aggressive. Should I allow for two collisions max?
486 */
487 if (keg->uk_flags & UMA_ZONE_HASH &&
488 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
489 struct uma_hash newhash;
490 struct uma_hash oldhash;
491 int ret;
492
493 /*
494 * This is so involved because allocating and freeing
495 * while the keg lock is held will lead to deadlock.
496 * I have to do everything in stages and check for
497 * races.
498 */
499 newhash = keg->uk_hash;
500 KEG_UNLOCK(keg);
501 ret = hash_alloc(&newhash);
502 KEG_LOCK(keg);
503 if (ret) {
504 if (hash_expand(&keg->uk_hash, &newhash)) {
505 oldhash = keg->uk_hash;
506 keg->uk_hash = newhash;
507 } else
508 oldhash = newhash;
509
510 KEG_UNLOCK(keg);
511 hash_free(&oldhash);
512 return;
513 }
514 }
515 KEG_UNLOCK(keg);
516 }
517
518 static void
zone_timeout(uma_zone_t zone)519 zone_timeout(uma_zone_t zone)
520 {
521
522 zone_foreach_keg(zone, &keg_timeout);
523 }
524
525 /*
526 * Allocate and zero fill the next sized hash table from the appropriate
527 * backing store.
528 *
529 * Arguments:
530 * hash A new hash structure with the old hash size in uh_hashsize
531 *
532 * Returns:
533 * 1 on sucess and 0 on failure.
534 */
535 static int
hash_alloc(struct uma_hash * hash)536 hash_alloc(struct uma_hash *hash)
537 {
538 int oldsize;
539 int alloc;
540
541 oldsize = hash->uh_hashsize;
542
543 /* We're just going to go to a power of two greater */
544 if (oldsize) {
545 hash->uh_hashsize = oldsize * 2;
546 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
547 hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
548 M_UMAHASH, M_NOWAIT);
549 } else {
550 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
551 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
552 M_WAITOK);
553 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
554 }
555 if (hash->uh_slab_hash) {
556 bzero(hash->uh_slab_hash, alloc);
557 hash->uh_hashmask = hash->uh_hashsize - 1;
558 return (1);
559 }
560
561 return (0);
562 }
563
564 /*
565 * Expands the hash table for HASH zones. This is done from zone_timeout
566 * to reduce collisions. This must not be done in the regular allocation
567 * path, otherwise, we can recurse on the vm while allocating pages.
568 *
569 * Arguments:
570 * oldhash The hash you want to expand
571 * newhash The hash structure for the new table
572 *
573 * Returns:
574 * Nothing
575 *
576 * Discussion:
577 */
578 static int
hash_expand(struct uma_hash * oldhash,struct uma_hash * newhash)579 hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
580 {
581 uma_slab_t slab;
582 int hval;
583 int i;
584
585 if (!newhash->uh_slab_hash)
586 return (0);
587
588 if (oldhash->uh_hashsize >= newhash->uh_hashsize)
589 return (0);
590
591 /*
592 * I need to investigate hash algorithms for resizing without a
593 * full rehash.
594 */
595
596 for (i = 0; i < oldhash->uh_hashsize; i++)
597 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
598 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
599 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
600 hval = UMA_HASH(newhash, slab->us_data);
601 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
602 slab, us_hlink);
603 }
604
605 return (1);
606 }
607
608 /*
609 * Free the hash bucket to the appropriate backing store.
610 *
611 * Arguments:
612 * slab_hash The hash bucket we're freeing
613 * hashsize The number of entries in that hash bucket
614 *
615 * Returns:
616 * Nothing
617 */
618 static void
hash_free(struct uma_hash * hash)619 hash_free(struct uma_hash *hash)
620 {
621 if (hash->uh_slab_hash == NULL)
622 return;
623 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
624 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
625 else
626 free(hash->uh_slab_hash, M_UMAHASH);
627 }
628
629 /*
630 * Frees all outstanding items in a bucket
631 *
632 * Arguments:
633 * zone The zone to free to, must be unlocked.
634 * bucket The free/alloc bucket with items, cpu queue must be locked.
635 *
636 * Returns:
637 * Nothing
638 */
639
640 static void
bucket_drain(uma_zone_t zone,uma_bucket_t bucket)641 bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
642 {
643 int i;
644
645 if (bucket == NULL)
646 return;
647
648 if (zone->uz_fini)
649 for (i = 0; i < bucket->ub_cnt; i++)
650 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
651 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
652 bucket->ub_cnt = 0;
653 }
654
655 /*
656 * Drains the per cpu caches for a zone.
657 *
658 * NOTE: This may only be called while the zone is being turn down, and not
659 * during normal operation. This is necessary in order that we do not have
660 * to migrate CPUs to drain the per-CPU caches.
661 *
662 * Arguments:
663 * zone The zone to drain, must be unlocked.
664 *
665 * Returns:
666 * Nothing
667 */
668 static void
cache_drain(uma_zone_t zone)669 cache_drain(uma_zone_t zone)
670 {
671 uma_cache_t cache;
672 int cpu;
673
674 /*
675 * XXX: It is safe to not lock the per-CPU caches, because we're
676 * tearing down the zone anyway. I.e., there will be no further use
677 * of the caches at this point.
678 *
679 * XXX: It would good to be able to assert that the zone is being
680 * torn down to prevent improper use of cache_drain().
681 *
682 * XXX: We lock the zone before passing into bucket_cache_drain() as
683 * it is used elsewhere. Should the tear-down path be made special
684 * there in some form?
685 */
686 CPU_FOREACH(cpu) {
687 cache = &zone->uz_cpu[cpu];
688 bucket_drain(zone, cache->uc_allocbucket);
689 bucket_drain(zone, cache->uc_freebucket);
690 if (cache->uc_allocbucket != NULL)
691 bucket_free(zone, cache->uc_allocbucket, NULL);
692 if (cache->uc_freebucket != NULL)
693 bucket_free(zone, cache->uc_freebucket, NULL);
694 cache->uc_allocbucket = cache->uc_freebucket = NULL;
695 }
696 ZONE_LOCK(zone);
697 bucket_cache_drain(zone);
698 ZONE_UNLOCK(zone);
699 }
700
701 static void
cache_shrink(uma_zone_t zone)702 cache_shrink(uma_zone_t zone)
703 {
704
705 if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
706 return;
707
708 ZONE_LOCK(zone);
709 zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
710 ZONE_UNLOCK(zone);
711 }
712
713 static void
cache_drain_safe_cpu(uma_zone_t zone)714 cache_drain_safe_cpu(uma_zone_t zone)
715 {
716 uma_cache_t cache;
717 uma_bucket_t b1, b2;
718
719 if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
720 return;
721
722 b1 = b2 = NULL;
723 ZONE_LOCK(zone);
724 critical_enter();
725 cache = &zone->uz_cpu[curcpu];
726 if (cache->uc_allocbucket) {
727 if (cache->uc_allocbucket->ub_cnt != 0)
728 LIST_INSERT_HEAD(&zone->uz_buckets,
729 cache->uc_allocbucket, ub_link);
730 else
731 b1 = cache->uc_allocbucket;
732 cache->uc_allocbucket = NULL;
733 }
734 if (cache->uc_freebucket) {
735 if (cache->uc_freebucket->ub_cnt != 0)
736 LIST_INSERT_HEAD(&zone->uz_buckets,
737 cache->uc_freebucket, ub_link);
738 else
739 b2 = cache->uc_freebucket;
740 cache->uc_freebucket = NULL;
741 }
742 critical_exit();
743 ZONE_UNLOCK(zone);
744 if (b1)
745 bucket_free(zone, b1, NULL);
746 if (b2)
747 bucket_free(zone, b2, NULL);
748 }
749
750 /*
751 * Safely drain per-CPU caches of a zone(s) to alloc bucket.
752 * This is an expensive call because it needs to bind to all CPUs
753 * one by one and enter a critical section on each of them in order
754 * to safely access their cache buckets.
755 * Zone lock must not be held on call this function.
756 */
757 static void
cache_drain_safe(uma_zone_t zone)758 cache_drain_safe(uma_zone_t zone)
759 {
760 int cpu;
761
762 /*
763 * Polite bucket sizes shrinking was not enouth, shrink aggressively.
764 */
765 if (zone)
766 cache_shrink(zone);
767 else
768 zone_foreach(cache_shrink);
769
770 CPU_FOREACH(cpu) {
771 thread_lock(curthread);
772 sched_bind(curthread, cpu);
773 thread_unlock(curthread);
774
775 if (zone)
776 cache_drain_safe_cpu(zone);
777 else
778 zone_foreach(cache_drain_safe_cpu);
779 }
780 thread_lock(curthread);
781 sched_unbind(curthread);
782 thread_unlock(curthread);
783 }
784
785 /*
786 * Drain the cached buckets from a zone. Expects a locked zone on entry.
787 */
788 static void
bucket_cache_drain(uma_zone_t zone)789 bucket_cache_drain(uma_zone_t zone)
790 {
791 uma_bucket_t bucket;
792
793 /*
794 * Drain the bucket queues and free the buckets, we just keep two per
795 * cpu (alloc/free).
796 */
797 while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
798 LIST_REMOVE(bucket, ub_link);
799 ZONE_UNLOCK(zone);
800 bucket_drain(zone, bucket);
801 bucket_free(zone, bucket, NULL);
802 ZONE_LOCK(zone);
803 }
804
805 /*
806 * Shrink further bucket sizes. Price of single zone lock collision
807 * is probably lower then price of global cache drain.
808 */
809 if (zone->uz_count > zone->uz_count_min)
810 zone->uz_count--;
811 }
812
813 static void
keg_free_slab(uma_keg_t keg,uma_slab_t slab,int start)814 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
815 {
816 uint8_t *mem;
817 int i;
818 uint8_t flags;
819
820 mem = slab->us_data;
821 flags = slab->us_flags;
822 i = start;
823 if (keg->uk_fini != NULL) {
824 for (i--; i > -1; i--)
825 keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
826 keg->uk_size);
827 }
828 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
829 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
830 #ifdef UMA_DEBUG
831 printf("%s: Returning %d bytes.\n", keg->uk_name,
832 PAGE_SIZE * keg->uk_ppera);
833 #endif
834 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
835 }
836
837 /*
838 * Frees pages from a keg back to the system. This is done on demand from
839 * the pageout daemon.
840 *
841 * Returns nothing.
842 */
843 static void
keg_drain(uma_keg_t keg)844 keg_drain(uma_keg_t keg)
845 {
846 struct slabhead freeslabs = { 0 };
847 uma_slab_t slab;
848 uma_slab_t n;
849
850 /*
851 * We don't want to take pages from statically allocated kegs at this
852 * time
853 */
854 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
855 return;
856
857 #ifdef UMA_DEBUG
858 printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
859 #endif
860 KEG_LOCK(keg);
861 if (keg->uk_free == 0)
862 goto finished;
863
864 slab = LIST_FIRST(&keg->uk_free_slab);
865 while (slab) {
866 n = LIST_NEXT(slab, us_link);
867
868 /* We have no where to free these to */
869 if (slab->us_flags & UMA_SLAB_BOOT) {
870 slab = n;
871 continue;
872 }
873
874 LIST_REMOVE(slab, us_link);
875 keg->uk_pages -= keg->uk_ppera;
876 keg->uk_free -= keg->uk_ipers;
877
878 if (keg->uk_flags & UMA_ZONE_HASH)
879 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
880
881 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
882
883 slab = n;
884 }
885 finished:
886 KEG_UNLOCK(keg);
887
888 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
889 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
890 keg_free_slab(keg, slab, keg->uk_ipers);
891 }
892 }
893
894 static void
zone_drain_wait(uma_zone_t zone,int waitok)895 zone_drain_wait(uma_zone_t zone, int waitok)
896 {
897
898 /*
899 * Set draining to interlock with zone_dtor() so we can release our
900 * locks as we go. Only dtor() should do a WAITOK call since it
901 * is the only call that knows the structure will still be available
902 * when it wakes up.
903 */
904 ZONE_LOCK(zone);
905 while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
906 if (waitok == M_NOWAIT)
907 goto out;
908 msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
909 }
910 zone->uz_flags |= UMA_ZFLAG_DRAINING;
911 bucket_cache_drain(zone);
912 ZONE_UNLOCK(zone);
913 /*
914 * The DRAINING flag protects us from being freed while
915 * we're running. Normally the uma_rwlock would protect us but we
916 * must be able to release and acquire the right lock for each keg.
917 */
918 zone_foreach_keg(zone, &keg_drain);
919 ZONE_LOCK(zone);
920 zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
921 wakeup(zone);
922 out:
923 ZONE_UNLOCK(zone);
924 }
925
926 void
zone_drain(uma_zone_t zone)927 zone_drain(uma_zone_t zone)
928 {
929
930 zone_drain_wait(zone, M_NOWAIT);
931 }
932
933 /*
934 * Allocate a new slab for a keg. This does not insert the slab onto a list.
935 *
936 * Arguments:
937 * wait Shall we wait?
938 *
939 * Returns:
940 * The slab that was allocated or NULL if there is no memory and the
941 * caller specified M_NOWAIT.
942 */
943 static uma_slab_t
keg_alloc_slab(uma_keg_t keg,uma_zone_t zone,int wait)944 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
945 {
946 uma_slabrefcnt_t slabref;
947 uma_alloc allocf;
948 uma_slab_t slab;
949 uint8_t *mem;
950 uint8_t flags;
951 int i;
952
953 mtx_assert(&keg->uk_lock, MA_OWNED);
954 slab = NULL;
955 mem = NULL;
956
957 #ifdef UMA_DEBUG
958 printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
959 #endif
960 allocf = keg->uk_allocf;
961 KEG_UNLOCK(keg);
962
963 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
964 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
965 if (slab == NULL)
966 goto out;
967 }
968
969 /*
970 * This reproduces the old vm_zone behavior of zero filling pages the
971 * first time they are added to a zone.
972 *
973 * Malloced items are zeroed in uma_zalloc.
974 */
975
976 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
977 wait |= M_ZERO;
978 else
979 wait &= ~M_ZERO;
980
981 if (keg->uk_flags & UMA_ZONE_NODUMP)
982 wait |= M_NODUMP;
983
984 /* zone is passed for legacy reasons. */
985 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
986 if (mem == NULL) {
987 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
988 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
989 slab = NULL;
990 goto out;
991 }
992
993 /* Point the slab into the allocated memory */
994 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
995 slab = (uma_slab_t )(mem + keg->uk_pgoff);
996
997 if (keg->uk_flags & UMA_ZONE_VTOSLAB)
998 for (i = 0; i < keg->uk_ppera; i++)
999 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
1000
1001 slab->us_keg = keg;
1002 slab->us_data = mem;
1003 slab->us_freecount = keg->uk_ipers;
1004 slab->us_flags = flags;
1005 BIT_FILL(SLAB_SETSIZE, &slab->us_free);
1006 #ifdef INVARIANTS
1007 BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
1008 #endif
1009 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1010 slabref = (uma_slabrefcnt_t)slab;
1011 for (i = 0; i < keg->uk_ipers; i++)
1012 slabref->us_refcnt[i] = 0;
1013 }
1014
1015 if (keg->uk_init != NULL) {
1016 for (i = 0; i < keg->uk_ipers; i++)
1017 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
1018 keg->uk_size, wait) != 0)
1019 break;
1020 if (i != keg->uk_ipers) {
1021 keg_free_slab(keg, slab, i);
1022 slab = NULL;
1023 goto out;
1024 }
1025 }
1026 out:
1027 KEG_LOCK(keg);
1028
1029 if (slab != NULL) {
1030 if (keg->uk_flags & UMA_ZONE_HASH)
1031 UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
1032
1033 keg->uk_pages += keg->uk_ppera;
1034 keg->uk_free += keg->uk_ipers;
1035 }
1036
1037 return (slab);
1038 }
1039
1040 /*
1041 * This function is intended to be used early on in place of page_alloc() so
1042 * that we may use the boot time page cache to satisfy allocations before
1043 * the VM is ready.
1044 */
1045 static void *
startup_alloc(uma_zone_t zone,vm_size_t bytes,uint8_t * pflag,int wait)1046 startup_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
1047 {
1048 uma_keg_t keg;
1049 uma_slab_t tmps;
1050 int pages, check_pages;
1051
1052 keg = zone_first_keg(zone);
1053 pages = howmany(bytes, PAGE_SIZE);
1054 check_pages = pages - 1;
1055 KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
1056
1057 /*
1058 * Check our small startup cache to see if it has pages remaining.
1059 */
1060 mtx_lock(&uma_boot_pages_mtx);
1061
1062 /* First check if we have enough room. */
1063 tmps = LIST_FIRST(&uma_boot_pages);
1064 while (tmps != NULL && check_pages-- > 0)
1065 tmps = LIST_NEXT(tmps, us_link);
1066 if (tmps != NULL) {
1067 /*
1068 * It's ok to lose tmps references. The last one will
1069 * have tmps->us_data pointing to the start address of
1070 * "pages" contiguous pages of memory.
1071 */
1072 while (pages-- > 0) {
1073 tmps = LIST_FIRST(&uma_boot_pages);
1074 LIST_REMOVE(tmps, us_link);
1075 }
1076 mtx_unlock(&uma_boot_pages_mtx);
1077 *pflag = tmps->us_flags;
1078 return (tmps->us_data);
1079 }
1080 mtx_unlock(&uma_boot_pages_mtx);
1081 if (booted < UMA_STARTUP2)
1082 panic("UMA: Increase vm.boot_pages");
1083 /*
1084 * Now that we've booted reset these users to their real allocator.
1085 */
1086 #ifdef UMA_MD_SMALL_ALLOC
1087 keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
1088 #else
1089 keg->uk_allocf = page_alloc;
1090 #endif
1091 return keg->uk_allocf(zone, bytes, pflag, wait);
1092 }
1093
1094 /*
1095 * Allocates a number of pages from the system
1096 *
1097 * Arguments:
1098 * bytes The number of bytes requested
1099 * wait Shall we wait?
1100 *
1101 * Returns:
1102 * A pointer to the alloced memory or possibly
1103 * NULL if M_NOWAIT is set.
1104 */
1105 static void *
page_alloc(uma_zone_t zone,vm_size_t bytes,uint8_t * pflag,int wait)1106 page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, int wait)
1107 {
1108 void *p; /* Returned page */
1109
1110 *pflag = UMA_SLAB_KMEM;
1111 p = (void *) kmem_malloc(kmem_arena, bytes, wait);
1112
1113 return (p);
1114 }
1115
1116 /*
1117 * Allocates a number of pages from within an object
1118 *
1119 * Arguments:
1120 * bytes The number of bytes requested
1121 * wait Shall we wait?
1122 *
1123 * Returns:
1124 * A pointer to the alloced memory or possibly
1125 * NULL if M_NOWAIT is set.
1126 */
1127 static void *
noobj_alloc(uma_zone_t zone,vm_size_t bytes,uint8_t * flags,int wait)1128 noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
1129 {
1130 TAILQ_HEAD(, vm_page) alloctail;
1131 u_long npages;
1132 vm_offset_t retkva, zkva;
1133 vm_page_t p, p_next;
1134 uma_keg_t keg;
1135
1136 TAILQ_INIT(&alloctail);
1137 keg = zone_first_keg(zone);
1138
1139 npages = howmany(bytes, PAGE_SIZE);
1140 while (npages > 0) {
1141 p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
1142 VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
1143 if (p != NULL) {
1144 /*
1145 * Since the page does not belong to an object, its
1146 * listq is unused.
1147 */
1148 TAILQ_INSERT_TAIL(&alloctail, p, listq);
1149 npages--;
1150 continue;
1151 }
1152 if (wait & M_WAITOK) {
1153 VM_WAIT;
1154 continue;
1155 }
1156
1157 /*
1158 * Page allocation failed, free intermediate pages and
1159 * exit.
1160 */
1161 TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1162 vm_page_unwire(p, PQ_NONE);
1163 vm_page_free(p);
1164 }
1165 return (NULL);
1166 }
1167 *flags = UMA_SLAB_PRIV;
1168 zkva = keg->uk_kva +
1169 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1170 retkva = zkva;
1171 TAILQ_FOREACH(p, &alloctail, listq) {
1172 pmap_qenter(zkva, &p, 1);
1173 zkva += PAGE_SIZE;
1174 }
1175
1176 return ((void *)retkva);
1177 }
1178
1179 /*
1180 * Frees a number of pages to the system
1181 *
1182 * Arguments:
1183 * mem A pointer to the memory to be freed
1184 * size The size of the memory being freed
1185 * flags The original p->us_flags field
1186 *
1187 * Returns:
1188 * Nothing
1189 */
1190 static void
page_free(void * mem,vm_size_t size,uint8_t flags)1191 page_free(void *mem, vm_size_t size, uint8_t flags)
1192 {
1193 struct vmem *vmem;
1194
1195 if (flags & UMA_SLAB_KMEM)
1196 vmem = kmem_arena;
1197 else if (flags & UMA_SLAB_KERNEL)
1198 vmem = kernel_arena;
1199 else
1200 panic("UMA: page_free used with invalid flags %d", flags);
1201
1202 kmem_free(vmem, (vm_offset_t)mem, size);
1203 }
1204
1205 /*
1206 * Zero fill initializer
1207 *
1208 * Arguments/Returns follow uma_init specifications
1209 */
1210 static int
zero_init(void * mem,int size,int flags)1211 zero_init(void *mem, int size, int flags)
1212 {
1213 bzero(mem, size);
1214 return (0);
1215 }
1216
1217 /*
1218 * Finish creating a small uma keg. This calculates ipers, and the keg size.
1219 *
1220 * Arguments
1221 * keg The zone we should initialize
1222 *
1223 * Returns
1224 * Nothing
1225 */
1226 static void
keg_small_init(uma_keg_t keg)1227 keg_small_init(uma_keg_t keg)
1228 {
1229 u_int rsize;
1230 u_int memused;
1231 u_int wastedspace;
1232 u_int shsize;
1233
1234 if (keg->uk_flags & UMA_ZONE_PCPU) {
1235 u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1236
1237 keg->uk_slabsize = sizeof(struct pcpu);
1238 keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu),
1239 PAGE_SIZE);
1240 } else {
1241 keg->uk_slabsize = UMA_SLAB_SIZE;
1242 keg->uk_ppera = 1;
1243 }
1244
1245 /*
1246 * Calculate the size of each allocation (rsize) according to
1247 * alignment. If the requested size is smaller than we have
1248 * allocation bits for we round it up.
1249 */
1250 rsize = keg->uk_size;
1251 if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
1252 rsize = keg->uk_slabsize / SLAB_SETSIZE;
1253 if (rsize & keg->uk_align)
1254 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1255 keg->uk_rsize = rsize;
1256
1257 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1258 keg->uk_rsize < sizeof(struct pcpu),
1259 ("%s: size %u too large", __func__, keg->uk_rsize));
1260
1261 if (keg->uk_flags & UMA_ZONE_REFCNT)
1262 rsize += sizeof(uint32_t);
1263
1264 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
1265 shsize = 0;
1266 else
1267 shsize = sizeof(struct uma_slab);
1268
1269 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1270 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1271 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1272
1273 memused = keg->uk_ipers * rsize + shsize;
1274 wastedspace = keg->uk_slabsize - memused;
1275
1276 /*
1277 * We can't do OFFPAGE if we're internal or if we've been
1278 * asked to not go to the VM for buckets. If we do this we
1279 * may end up going to the VM for slabs which we do not
1280 * want to do if we're UMA_ZFLAG_CACHEONLY as a result
1281 * of UMA_ZONE_VM, which clearly forbids it.
1282 */
1283 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1284 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1285 return;
1286
1287 /*
1288 * See if using an OFFPAGE slab will limit our waste. Only do
1289 * this if it permits more items per-slab.
1290 *
1291 * XXX We could try growing slabsize to limit max waste as well.
1292 * Historically this was not done because the VM could not
1293 * efficiently handle contiguous allocations.
1294 */
1295 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1296 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1297 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1298 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1299 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1300 #ifdef UMA_DEBUG
1301 printf("UMA decided we need offpage slab headers for "
1302 "keg: %s, calculated wastedspace = %d, "
1303 "maximum wasted space allowed = %d, "
1304 "calculated ipers = %d, "
1305 "new wasted space = %d\n", keg->uk_name, wastedspace,
1306 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1307 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1308 #endif
1309 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1310 }
1311
1312 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1313 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1314 keg->uk_flags |= UMA_ZONE_HASH;
1315 }
1316
1317 /*
1318 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1319 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1320 * more complicated.
1321 *
1322 * Arguments
1323 * keg The keg we should initialize
1324 *
1325 * Returns
1326 * Nothing
1327 */
1328 static void
keg_large_init(uma_keg_t keg)1329 keg_large_init(uma_keg_t keg)
1330 {
1331 u_int shsize;
1332
1333 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1334 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1335 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1336 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1337 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
1338
1339 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1340 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1341 keg->uk_ipers = 1;
1342 keg->uk_rsize = keg->uk_size;
1343
1344 /* We can't do OFFPAGE if we're internal, bail out here. */
1345 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1346 return;
1347
1348 /* Check whether we have enough space to not do OFFPAGE. */
1349 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) {
1350 shsize = sizeof(struct uma_slab);
1351 if (keg->uk_flags & UMA_ZONE_REFCNT)
1352 shsize += keg->uk_ipers * sizeof(uint32_t);
1353 if (shsize & UMA_ALIGN_PTR)
1354 shsize = (shsize & ~UMA_ALIGN_PTR) +
1355 (UMA_ALIGN_PTR + 1);
1356
1357 if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize)
1358 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1359 }
1360
1361 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1362 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1363 keg->uk_flags |= UMA_ZONE_HASH;
1364 }
1365
1366 static void
keg_cachespread_init(uma_keg_t keg)1367 keg_cachespread_init(uma_keg_t keg)
1368 {
1369 int alignsize;
1370 int trailer;
1371 int pages;
1372 int rsize;
1373
1374 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1375 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1376
1377 alignsize = keg->uk_align + 1;
1378 rsize = keg->uk_size;
1379 /*
1380 * We want one item to start on every align boundary in a page. To
1381 * do this we will span pages. We will also extend the item by the
1382 * size of align if it is an even multiple of align. Otherwise, it
1383 * would fall on the same boundary every time.
1384 */
1385 if (rsize & keg->uk_align)
1386 rsize = (rsize & ~keg->uk_align) + alignsize;
1387 if ((rsize & alignsize) == 0)
1388 rsize += alignsize;
1389 trailer = rsize - keg->uk_size;
1390 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1391 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1392 keg->uk_rsize = rsize;
1393 keg->uk_ppera = pages;
1394 keg->uk_slabsize = UMA_SLAB_SIZE;
1395 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1396 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1397 KASSERT(keg->uk_ipers <= SLAB_SETSIZE,
1398 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1399 keg->uk_ipers));
1400 }
1401
1402 /*
1403 * Keg header ctor. This initializes all fields, locks, etc. And inserts
1404 * the keg onto the global keg list.
1405 *
1406 * Arguments/Returns follow uma_ctor specifications
1407 * udata Actually uma_kctor_args
1408 */
1409 static int
keg_ctor(void * mem,int size,void * udata,int flags)1410 keg_ctor(void *mem, int size, void *udata, int flags)
1411 {
1412 struct uma_kctor_args *arg = udata;
1413 uma_keg_t keg = mem;
1414 uma_zone_t zone;
1415
1416 bzero(keg, size);
1417 keg->uk_size = arg->size;
1418 keg->uk_init = arg->uminit;
1419 keg->uk_fini = arg->fini;
1420 keg->uk_align = arg->align;
1421 keg->uk_free = 0;
1422 keg->uk_reserve = 0;
1423 keg->uk_pages = 0;
1424 keg->uk_flags = arg->flags;
1425 keg->uk_allocf = page_alloc;
1426 keg->uk_freef = page_free;
1427 keg->uk_slabzone = NULL;
1428
1429 /*
1430 * The master zone is passed to us at keg-creation time.
1431 */
1432 zone = arg->zone;
1433 keg->uk_name = zone->uz_name;
1434
1435 if (arg->flags & UMA_ZONE_VM)
1436 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1437
1438 if (arg->flags & UMA_ZONE_ZINIT)
1439 keg->uk_init = zero_init;
1440
1441 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1442 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1443
1444 if (arg->flags & UMA_ZONE_PCPU)
1445 #ifdef SMP
1446 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1447 #else
1448 keg->uk_flags &= ~UMA_ZONE_PCPU;
1449 #endif
1450
1451 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
1452 keg_cachespread_init(keg);
1453 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1454 if (keg->uk_size >
1455 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1456 sizeof(uint32_t)))
1457 keg_large_init(keg);
1458 else
1459 keg_small_init(keg);
1460 } else {
1461 if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
1462 keg_large_init(keg);
1463 else
1464 keg_small_init(keg);
1465 }
1466
1467 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1468 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1469 if (keg->uk_ipers > uma_max_ipers_ref)
1470 panic("Too many ref items per zone: %d > %d\n",
1471 keg->uk_ipers, uma_max_ipers_ref);
1472 keg->uk_slabzone = slabrefzone;
1473 } else
1474 keg->uk_slabzone = slabzone;
1475 }
1476
1477 /*
1478 * If we haven't booted yet we need allocations to go through the
1479 * startup cache until the vm is ready.
1480 */
1481 if (keg->uk_ppera == 1) {
1482 #ifdef UMA_MD_SMALL_ALLOC
1483 keg->uk_allocf = uma_small_alloc;
1484 keg->uk_freef = uma_small_free;
1485
1486 if (booted < UMA_STARTUP)
1487 keg->uk_allocf = startup_alloc;
1488 #else
1489 if (booted < UMA_STARTUP2)
1490 keg->uk_allocf = startup_alloc;
1491 #endif
1492 } else if (booted < UMA_STARTUP2 &&
1493 (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1494 keg->uk_allocf = startup_alloc;
1495
1496 /*
1497 * Initialize keg's lock
1498 */
1499 KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
1500
1501 /*
1502 * If we're putting the slab header in the actual page we need to
1503 * figure out where in each page it goes. This calculates a right
1504 * justified offset into the memory on an ALIGN_PTR boundary.
1505 */
1506 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1507 u_int totsize;
1508
1509 /* Size of the slab struct and free list */
1510 totsize = sizeof(struct uma_slab);
1511
1512 /* Size of the reference counts. */
1513 if (keg->uk_flags & UMA_ZONE_REFCNT)
1514 totsize += keg->uk_ipers * sizeof(uint32_t);
1515
1516 if (totsize & UMA_ALIGN_PTR)
1517 totsize = (totsize & ~UMA_ALIGN_PTR) +
1518 (UMA_ALIGN_PTR + 1);
1519 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1520
1521 /*
1522 * The only way the following is possible is if with our
1523 * UMA_ALIGN_PTR adjustments we are now bigger than
1524 * UMA_SLAB_SIZE. I haven't checked whether this is
1525 * mathematically possible for all cases, so we make
1526 * sure here anyway.
1527 */
1528 totsize = keg->uk_pgoff + sizeof(struct uma_slab);
1529 if (keg->uk_flags & UMA_ZONE_REFCNT)
1530 totsize += keg->uk_ipers * sizeof(uint32_t);
1531 if (totsize > PAGE_SIZE * keg->uk_ppera) {
1532 printf("zone %s ipers %d rsize %d size %d\n",
1533 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1534 keg->uk_size);
1535 panic("UMA slab won't fit.");
1536 }
1537 }
1538
1539 if (keg->uk_flags & UMA_ZONE_HASH)
1540 hash_alloc(&keg->uk_hash);
1541
1542 #ifdef UMA_DEBUG
1543 printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
1544 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
1545 keg->uk_ipers, keg->uk_ppera,
1546 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
1547 #endif
1548
1549 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
1550
1551 rw_wlock(&uma_rwlock);
1552 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
1553 rw_wunlock(&uma_rwlock);
1554 return (0);
1555 }
1556
1557 /*
1558 * Zone header ctor. This initializes all fields, locks, etc.
1559 *
1560 * Arguments/Returns follow uma_ctor specifications
1561 * udata Actually uma_zctor_args
1562 */
1563 static int
zone_ctor(void * mem,int size,void * udata,int flags)1564 zone_ctor(void *mem, int size, void *udata, int flags)
1565 {
1566 struct uma_zctor_args *arg = udata;
1567 uma_zone_t zone = mem;
1568 uma_zone_t z;
1569 uma_keg_t keg;
1570
1571 bzero(zone, size);
1572 zone->uz_name = arg->name;
1573 zone->uz_ctor = arg->ctor;
1574 zone->uz_dtor = arg->dtor;
1575 zone->uz_slab = zone_fetch_slab;
1576 zone->uz_init = NULL;
1577 zone->uz_fini = NULL;
1578 zone->uz_allocs = 0;
1579 zone->uz_frees = 0;
1580 zone->uz_fails = 0;
1581 zone->uz_sleeps = 0;
1582 zone->uz_count = 0;
1583 zone->uz_count_min = 0;
1584 zone->uz_flags = 0;
1585 zone->uz_warning = NULL;
1586 timevalclear(&zone->uz_ratecheck);
1587 zone->uz_maxaction = NULL;
1588 keg = arg->keg;
1589
1590 ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
1591
1592 /*
1593 * This is a pure cache zone, no kegs.
1594 */
1595 if (arg->import) {
1596 if (arg->flags & UMA_ZONE_VM)
1597 arg->flags |= UMA_ZFLAG_CACHEONLY;
1598 zone->uz_flags = arg->flags;
1599 zone->uz_size = arg->size;
1600 zone->uz_import = arg->import;
1601 zone->uz_release = arg->release;
1602 zone->uz_arg = arg->arg;
1603 zone->uz_lockptr = &zone->uz_lock;
1604 rw_wlock(&uma_rwlock);
1605 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
1606 rw_wunlock(&uma_rwlock);
1607 goto out;
1608 }
1609
1610 /*
1611 * Use the regular zone/keg/slab allocator.
1612 */
1613 zone->uz_import = (uma_import)zone_import;
1614 zone->uz_release = (uma_release)zone_release;
1615 zone->uz_arg = zone;
1616
1617 if (arg->flags & UMA_ZONE_SECONDARY) {
1618 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1619 zone->uz_init = arg->uminit;
1620 zone->uz_fini = arg->fini;
1621 zone->uz_lockptr = &keg->uk_lock;
1622 zone->uz_flags |= UMA_ZONE_SECONDARY;
1623 rw_wlock(&uma_rwlock);
1624 ZONE_LOCK(zone);
1625 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1626 if (LIST_NEXT(z, uz_link) == NULL) {
1627 LIST_INSERT_AFTER(z, zone, uz_link);
1628 break;
1629 }
1630 }
1631 ZONE_UNLOCK(zone);
1632 rw_wunlock(&uma_rwlock);
1633 } else if (keg == NULL) {
1634 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
1635 arg->align, arg->flags)) == NULL)
1636 return (ENOMEM);
1637 } else {
1638 struct uma_kctor_args karg;
1639 int error;
1640
1641 /* We should only be here from uma_startup() */
1642 karg.size = arg->size;
1643 karg.uminit = arg->uminit;
1644 karg.fini = arg->fini;
1645 karg.align = arg->align;
1646 karg.flags = arg->flags;
1647 karg.zone = zone;
1648 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1649 flags);
1650 if (error)
1651 return (error);
1652 }
1653
1654 /*
1655 * Link in the first keg.
1656 */
1657 zone->uz_klink.kl_keg = keg;
1658 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1659 zone->uz_lockptr = &keg->uk_lock;
1660 zone->uz_size = keg->uk_size;
1661 zone->uz_flags |= (keg->uk_flags &
1662 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
1663
1664 /*
1665 * Some internal zones don't have room allocated for the per cpu
1666 * caches. If we're internal, bail out here.
1667 */
1668 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1669 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1670 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1671 return (0);
1672 }
1673
1674 out:
1675 if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
1676 zone->uz_count = bucket_select(zone->uz_size);
1677 else
1678 zone->uz_count = BUCKET_MAX;
1679 zone->uz_count_min = zone->uz_count;
1680
1681 return (0);
1682 }
1683
1684 /*
1685 * Keg header dtor. This frees all data, destroys locks, frees the hash
1686 * table and removes the keg from the global list.
1687 *
1688 * Arguments/Returns follow uma_dtor specifications
1689 * udata unused
1690 */
1691 static void
keg_dtor(void * arg,int size,void * udata)1692 keg_dtor(void *arg, int size, void *udata)
1693 {
1694 uma_keg_t keg;
1695
1696 keg = (uma_keg_t)arg;
1697 KEG_LOCK(keg);
1698 if (keg->uk_free != 0) {
1699 printf("Freed UMA keg (%s) was not empty (%d items). "
1700 " Lost %d pages of memory.\n",
1701 keg->uk_name ? keg->uk_name : "",
1702 keg->uk_free, keg->uk_pages);
1703 }
1704 KEG_UNLOCK(keg);
1705
1706 hash_free(&keg->uk_hash);
1707
1708 KEG_LOCK_FINI(keg);
1709 }
1710
1711 /*
1712 * Zone header dtor.
1713 *
1714 * Arguments/Returns follow uma_dtor specifications
1715 * udata unused
1716 */
1717 static void
zone_dtor(void * arg,int size,void * udata)1718 zone_dtor(void *arg, int size, void *udata)
1719 {
1720 uma_klink_t klink;
1721 uma_zone_t zone;
1722 uma_keg_t keg;
1723
1724 zone = (uma_zone_t)arg;
1725 keg = zone_first_keg(zone);
1726
1727 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
1728 cache_drain(zone);
1729
1730 rw_wlock(&uma_rwlock);
1731 LIST_REMOVE(zone, uz_link);
1732 rw_wunlock(&uma_rwlock);
1733 /*
1734 * XXX there are some races here where
1735 * the zone can be drained but zone lock
1736 * released and then refilled before we
1737 * remove it... we dont care for now
1738 */
1739 zone_drain_wait(zone, M_WAITOK);
1740 /*
1741 * Unlink all of our kegs.
1742 */
1743 while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
1744 klink->kl_keg = NULL;
1745 LIST_REMOVE(klink, kl_link);
1746 if (klink == &zone->uz_klink)
1747 continue;
1748 free(klink, M_TEMP);
1749 }
1750 /*
1751 * We only destroy kegs from non secondary zones.
1752 */
1753 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1754 rw_wlock(&uma_rwlock);
1755 LIST_REMOVE(keg, uk_link);
1756 rw_wunlock(&uma_rwlock);
1757 zone_free_item(kegs, keg, NULL, SKIP_NONE);
1758 }
1759 ZONE_LOCK_FINI(zone);
1760 }
1761
1762 /*
1763 * Traverses every zone in the system and calls a callback
1764 *
1765 * Arguments:
1766 * zfunc A pointer to a function which accepts a zone
1767 * as an argument.
1768 *
1769 * Returns:
1770 * Nothing
1771 */
1772 static void
zone_foreach(void (* zfunc)(uma_zone_t))1773 zone_foreach(void (*zfunc)(uma_zone_t))
1774 {
1775 uma_keg_t keg;
1776 uma_zone_t zone;
1777
1778 rw_rlock(&uma_rwlock);
1779 LIST_FOREACH(keg, &uma_kegs, uk_link) {
1780 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
1781 zfunc(zone);
1782 }
1783 rw_runlock(&uma_rwlock);
1784 }
1785
1786 /* Public functions */
1787 /* See uma.h */
1788 void
uma_startup(void * bootmem,int boot_pages)1789 uma_startup(void *bootmem, int boot_pages)
1790 {
1791 struct uma_zctor_args args;
1792 uma_slab_t slab;
1793 u_int slabsize;
1794 int i;
1795
1796 #ifdef UMA_DEBUG
1797 printf("Creating uma keg headers zone and keg.\n");
1798 #endif
1799 rw_init(&uma_rwlock, "UMA lock");
1800
1801 /* "manually" create the initial zone */
1802 memset(&args, 0, sizeof(args));
1803 args.name = "UMA Kegs";
1804 args.size = sizeof(struct uma_keg);
1805 args.ctor = keg_ctor;
1806 args.dtor = keg_dtor;
1807 args.uminit = zero_init;
1808 args.fini = NULL;
1809 args.keg = &masterkeg;
1810 args.align = 32 - 1;
1811 args.flags = UMA_ZFLAG_INTERNAL;
1812 /* The initial zone has no Per cpu queues so it's smaller */
1813 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
1814
1815 #ifdef UMA_DEBUG
1816 printf("Filling boot free list.\n");
1817 #endif
1818 for (i = 0; i < boot_pages; i++) {
1819 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
1820 slab->us_data = (uint8_t *)slab;
1821 slab->us_flags = UMA_SLAB_BOOT;
1822 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
1823 }
1824 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
1825
1826 #ifdef UMA_DEBUG
1827 printf("Creating uma zone headers zone and keg.\n");
1828 #endif
1829 args.name = "UMA Zones";
1830 args.size = sizeof(struct uma_zone) +
1831 (sizeof(struct uma_cache) * (mp_maxid + 1));
1832 args.ctor = zone_ctor;
1833 args.dtor = zone_dtor;
1834 args.uminit = zero_init;
1835 args.fini = NULL;
1836 args.keg = NULL;
1837 args.align = 32 - 1;
1838 args.flags = UMA_ZFLAG_INTERNAL;
1839 /* The initial zone has no Per cpu queues so it's smaller */
1840 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
1841
1842 #ifdef UMA_DEBUG
1843 printf("Creating slab and hash zones.\n");
1844 #endif
1845
1846 /* Now make a zone for slab headers */
1847 slabzone = uma_zcreate("UMA Slabs",
1848 sizeof(struct uma_slab),
1849 NULL, NULL, NULL, NULL,
1850 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1851
1852 /*
1853 * We also create a zone for the bigger slabs with reference
1854 * counts in them, to accomodate UMA_ZONE_REFCNT zones.
1855 */
1856 slabsize = sizeof(struct uma_slab_refcnt);
1857 slabsize += uma_max_ipers_ref * sizeof(uint32_t);
1858 slabrefzone = uma_zcreate("UMA RCntSlabs",
1859 slabsize,
1860 NULL, NULL, NULL, NULL,
1861 UMA_ALIGN_PTR,
1862 UMA_ZFLAG_INTERNAL);
1863
1864 hashzone = uma_zcreate("UMA Hash",
1865 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
1866 NULL, NULL, NULL, NULL,
1867 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1868
1869 bucket_init();
1870
1871 booted = UMA_STARTUP;
1872
1873 #ifdef UMA_DEBUG
1874 printf("UMA startup complete.\n");
1875 #endif
1876 }
1877
1878 /* see uma.h */
1879 void
uma_startup2(void)1880 uma_startup2(void)
1881 {
1882 booted = UMA_STARTUP2;
1883 bucket_enable();
1884 sx_init(&uma_drain_lock, "umadrain");
1885 #ifdef UMA_DEBUG
1886 printf("UMA startup2 complete.\n");
1887 #endif
1888 }
1889
1890 /*
1891 * Initialize our callout handle
1892 *
1893 */
1894
1895 static void
uma_startup3(void)1896 uma_startup3(void)
1897 {
1898 #ifdef UMA_DEBUG
1899 printf("Starting callout.\n");
1900 #endif
1901 callout_init(&uma_callout, 1);
1902 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
1903 #ifdef UMA_DEBUG
1904 printf("UMA startup3 complete.\n");
1905 #endif
1906 }
1907
1908 static uma_keg_t
uma_kcreate(uma_zone_t zone,size_t size,uma_init uminit,uma_fini fini,int align,uint32_t flags)1909 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
1910 int align, uint32_t flags)
1911 {
1912 struct uma_kctor_args args;
1913
1914 args.size = size;
1915 args.uminit = uminit;
1916 args.fini = fini;
1917 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1918 args.flags = flags;
1919 args.zone = zone;
1920 return (zone_alloc_item(kegs, &args, M_WAITOK));
1921 }
1922
1923 /* See uma.h */
1924 void
uma_set_align(int align)1925 uma_set_align(int align)
1926 {
1927
1928 if (align != UMA_ALIGN_CACHE)
1929 uma_align_cache = align;
1930 }
1931
1932 /* See uma.h */
1933 uma_zone_t
uma_zcreate(const char * name,size_t size,uma_ctor ctor,uma_dtor dtor,uma_init uminit,uma_fini fini,int align,uint32_t flags)1934 uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1935 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1936
1937 {
1938 struct uma_zctor_args args;
1939 uma_zone_t res;
1940 bool locked;
1941
1942 /* This stuff is essential for the zone ctor */
1943 memset(&args, 0, sizeof(args));
1944 args.name = name;
1945 args.size = size;
1946 args.ctor = ctor;
1947 args.dtor = dtor;
1948 args.uminit = uminit;
1949 args.fini = fini;
1950 #ifdef INVARIANTS
1951 /*
1952 * If a zone is being created with an empty constructor and
1953 * destructor, pass UMA constructor/destructor which checks for
1954 * memory use after free.
1955 */
1956 if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) &&
1957 ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) {
1958 args.ctor = trash_ctor;
1959 args.dtor = trash_dtor;
1960 args.uminit = trash_init;
1961 args.fini = trash_fini;
1962 }
1963 #endif
1964 args.align = align;
1965 args.flags = flags;
1966 args.keg = NULL;
1967
1968 if (booted < UMA_STARTUP2) {
1969 locked = false;
1970 } else {
1971 sx_slock(&uma_drain_lock);
1972 locked = true;
1973 }
1974 res = zone_alloc_item(zones, &args, M_WAITOK);
1975 if (locked)
1976 sx_sunlock(&uma_drain_lock);
1977 return (res);
1978 }
1979
1980 /* See uma.h */
1981 uma_zone_t
uma_zsecond_create(char * name,uma_ctor ctor,uma_dtor dtor,uma_init zinit,uma_fini zfini,uma_zone_t master)1982 uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1983 uma_init zinit, uma_fini zfini, uma_zone_t master)
1984 {
1985 struct uma_zctor_args args;
1986 uma_keg_t keg;
1987 uma_zone_t res;
1988 bool locked;
1989
1990 keg = zone_first_keg(master);
1991 memset(&args, 0, sizeof(args));
1992 args.name = name;
1993 args.size = keg->uk_size;
1994 args.ctor = ctor;
1995 args.dtor = dtor;
1996 args.uminit = zinit;
1997 args.fini = zfini;
1998 args.align = keg->uk_align;
1999 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
2000 args.keg = keg;
2001
2002 if (booted < UMA_STARTUP2) {
2003 locked = false;
2004 } else {
2005 sx_slock(&uma_drain_lock);
2006 locked = true;
2007 }
2008 /* XXX Attaches only one keg of potentially many. */
2009 res = zone_alloc_item(zones, &args, M_WAITOK);
2010 if (locked)
2011 sx_sunlock(&uma_drain_lock);
2012 return (res);
2013 }
2014
2015 /* See uma.h */
2016 uma_zone_t
uma_zcache_create(char * name,int size,uma_ctor ctor,uma_dtor dtor,uma_init zinit,uma_fini zfini,uma_import zimport,uma_release zrelease,void * arg,int flags)2017 uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
2018 uma_init zinit, uma_fini zfini, uma_import zimport,
2019 uma_release zrelease, void *arg, int flags)
2020 {
2021 struct uma_zctor_args args;
2022
2023 memset(&args, 0, sizeof(args));
2024 args.name = name;
2025 args.size = size;
2026 args.ctor = ctor;
2027 args.dtor = dtor;
2028 args.uminit = zinit;
2029 args.fini = zfini;
2030 args.import = zimport;
2031 args.release = zrelease;
2032 args.arg = arg;
2033 args.align = 0;
2034 args.flags = flags;
2035
2036 return (zone_alloc_item(zones, &args, M_WAITOK));
2037 }
2038
2039 static void
zone_lock_pair(uma_zone_t a,uma_zone_t b)2040 zone_lock_pair(uma_zone_t a, uma_zone_t b)
2041 {
2042 if (a < b) {
2043 ZONE_LOCK(a);
2044 mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
2045 } else {
2046 ZONE_LOCK(b);
2047 mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
2048 }
2049 }
2050
2051 static void
zone_unlock_pair(uma_zone_t a,uma_zone_t b)2052 zone_unlock_pair(uma_zone_t a, uma_zone_t b)
2053 {
2054
2055 ZONE_UNLOCK(a);
2056 ZONE_UNLOCK(b);
2057 }
2058
2059 int
uma_zsecond_add(uma_zone_t zone,uma_zone_t master)2060 uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
2061 {
2062 uma_klink_t klink;
2063 uma_klink_t kl;
2064 int error;
2065
2066 error = 0;
2067 klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
2068
2069 zone_lock_pair(zone, master);
2070 /*
2071 * zone must use vtoslab() to resolve objects and must already be
2072 * a secondary.
2073 */
2074 if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
2075 != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
2076 error = EINVAL;
2077 goto out;
2078 }
2079 /*
2080 * The new master must also use vtoslab().
2081 */
2082 if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
2083 error = EINVAL;
2084 goto out;
2085 }
2086 /*
2087 * Both must either be refcnt, or not be refcnt.
2088 */
2089 if ((zone->uz_flags & UMA_ZONE_REFCNT) !=
2090 (master->uz_flags & UMA_ZONE_REFCNT)) {
2091 error = EINVAL;
2092 goto out;
2093 }
2094 /*
2095 * The underlying object must be the same size. rsize
2096 * may be different.
2097 */
2098 if (master->uz_size != zone->uz_size) {
2099 error = E2BIG;
2100 goto out;
2101 }
2102 /*
2103 * Put it at the end of the list.
2104 */
2105 klink->kl_keg = zone_first_keg(master);
2106 LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
2107 if (LIST_NEXT(kl, kl_link) == NULL) {
2108 LIST_INSERT_AFTER(kl, klink, kl_link);
2109 break;
2110 }
2111 }
2112 klink = NULL;
2113 zone->uz_flags |= UMA_ZFLAG_MULTI;
2114 zone->uz_slab = zone_fetch_slab_multi;
2115
2116 out:
2117 zone_unlock_pair(zone, master);
2118 if (klink != NULL)
2119 free(klink, M_TEMP);
2120
2121 return (error);
2122 }
2123
2124
2125 /* See uma.h */
2126 void
uma_zdestroy(uma_zone_t zone)2127 uma_zdestroy(uma_zone_t zone)
2128 {
2129
2130 sx_slock(&uma_drain_lock);
2131 zone_free_item(zones, zone, NULL, SKIP_NONE);
2132 sx_sunlock(&uma_drain_lock);
2133 }
2134
2135 /* See uma.h */
2136 void *
uma_zalloc_arg(uma_zone_t zone,void * udata,int flags)2137 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
2138 {
2139 void *item;
2140 uma_cache_t cache;
2141 uma_bucket_t bucket;
2142 int lockfail;
2143 int cpu;
2144
2145 /* This is the fast path allocation */
2146 #ifdef UMA_DEBUG_ALLOC_1
2147 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
2148 #endif
2149 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
2150 zone->uz_name, flags);
2151
2152 if (flags & M_WAITOK) {
2153 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2154 "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
2155 }
2156 KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
2157 ("uma_zalloc_arg: called with spinlock or critical section held"));
2158
2159 #ifdef DEBUG_MEMGUARD
2160 if (memguard_cmp_zone(zone)) {
2161 item = memguard_alloc(zone->uz_size, flags);
2162 if (item != NULL) {
2163 /*
2164 * Avoid conflict with the use-after-free
2165 * protecting infrastructure from INVARIANTS.
2166 */
2167 if (zone->uz_init != NULL &&
2168 zone->uz_init != mtrash_init &&
2169 zone->uz_init(item, zone->uz_size, flags) != 0)
2170 return (NULL);
2171 if (zone->uz_ctor != NULL &&
2172 zone->uz_ctor != mtrash_ctor &&
2173 zone->uz_ctor(item, zone->uz_size, udata,
2174 flags) != 0) {
2175 zone->uz_fini(item, zone->uz_size);
2176 return (NULL);
2177 }
2178 return (item);
2179 }
2180 /* This is unfortunate but should not be fatal. */
2181 }
2182 #endif
2183 /*
2184 * If possible, allocate from the per-CPU cache. There are two
2185 * requirements for safe access to the per-CPU cache: (1) the thread
2186 * accessing the cache must not be preempted or yield during access,
2187 * and (2) the thread must not migrate CPUs without switching which
2188 * cache it accesses. We rely on a critical section to prevent
2189 * preemption and migration. We release the critical section in
2190 * order to acquire the zone mutex if we are unable to allocate from
2191 * the current cache; when we re-acquire the critical section, we
2192 * must detect and handle migration if it has occurred.
2193 */
2194 critical_enter();
2195 cpu = curcpu;
2196 cache = &zone->uz_cpu[cpu];
2197
2198 zalloc_start:
2199 bucket = cache->uc_allocbucket;
2200 if (bucket != NULL && bucket->ub_cnt > 0) {
2201 bucket->ub_cnt--;
2202 item = bucket->ub_bucket[bucket->ub_cnt];
2203 #ifdef INVARIANTS
2204 bucket->ub_bucket[bucket->ub_cnt] = NULL;
2205 #endif
2206 KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
2207 cache->uc_allocs++;
2208 critical_exit();
2209 if (zone->uz_ctor != NULL &&
2210 zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2211 atomic_add_long(&zone->uz_fails, 1);
2212 zone_free_item(zone, item, udata, SKIP_DTOR);
2213 return (NULL);
2214 }
2215 #ifdef INVARIANTS
2216 uma_dbg_alloc(zone, NULL, item);
2217 #endif
2218 if (flags & M_ZERO)
2219 uma_zero_item(item, zone);
2220 return (item);
2221 }
2222
2223 /*
2224 * We have run out of items in our alloc bucket.
2225 * See if we can switch with our free bucket.
2226 */
2227 bucket = cache->uc_freebucket;
2228 if (bucket != NULL && bucket->ub_cnt > 0) {
2229 #ifdef UMA_DEBUG_ALLOC
2230 printf("uma_zalloc: Swapping empty with alloc.\n");
2231 #endif
2232 cache->uc_freebucket = cache->uc_allocbucket;
2233 cache->uc_allocbucket = bucket;
2234 goto zalloc_start;
2235 }
2236
2237 /*
2238 * Discard any empty allocation bucket while we hold no locks.
2239 */
2240 bucket = cache->uc_allocbucket;
2241 cache->uc_allocbucket = NULL;
2242 critical_exit();
2243 if (bucket != NULL)
2244 bucket_free(zone, bucket, udata);
2245
2246 /* Short-circuit for zones without buckets and low memory. */
2247 if (zone->uz_count == 0 || bucketdisable)
2248 goto zalloc_item;
2249
2250 /*
2251 * Attempt to retrieve the item from the per-CPU cache has failed, so
2252 * we must go back to the zone. This requires the zone lock, so we
2253 * must drop the critical section, then re-acquire it when we go back
2254 * to the cache. Since the critical section is released, we may be
2255 * preempted or migrate. As such, make sure not to maintain any
2256 * thread-local state specific to the cache from prior to releasing
2257 * the critical section.
2258 */
2259 lockfail = 0;
2260 if (ZONE_TRYLOCK(zone) == 0) {
2261 /* Record contention to size the buckets. */
2262 ZONE_LOCK(zone);
2263 lockfail = 1;
2264 }
2265 critical_enter();
2266 cpu = curcpu;
2267 cache = &zone->uz_cpu[cpu];
2268
2269 /*
2270 * Since we have locked the zone we may as well send back our stats.
2271 */
2272 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2273 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2274 cache->uc_allocs = 0;
2275 cache->uc_frees = 0;
2276
2277 /* See if we lost the race to fill the cache. */
2278 if (cache->uc_allocbucket != NULL) {
2279 ZONE_UNLOCK(zone);
2280 goto zalloc_start;
2281 }
2282
2283 /*
2284 * Check the zone's cache of buckets.
2285 */
2286 if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
2287 KASSERT(bucket->ub_cnt != 0,
2288 ("uma_zalloc_arg: Returning an empty bucket."));
2289
2290 LIST_REMOVE(bucket, ub_link);
2291 cache->uc_allocbucket = bucket;
2292 ZONE_UNLOCK(zone);
2293 goto zalloc_start;
2294 }
2295 /* We are no longer associated with this CPU. */
2296 critical_exit();
2297
2298 /*
2299 * We bump the uz count when the cache size is insufficient to
2300 * handle the working set.
2301 */
2302 if (lockfail && zone->uz_count < BUCKET_MAX)
2303 zone->uz_count++;
2304 ZONE_UNLOCK(zone);
2305
2306 /*
2307 * Now lets just fill a bucket and put it on the free list. If that
2308 * works we'll restart the allocation from the begining and it
2309 * will use the just filled bucket.
2310 */
2311 bucket = zone_alloc_bucket(zone, udata, flags);
2312 if (bucket != NULL) {
2313 ZONE_LOCK(zone);
2314 critical_enter();
2315 cpu = curcpu;
2316 cache = &zone->uz_cpu[cpu];
2317 /*
2318 * See if we lost the race or were migrated. Cache the
2319 * initialized bucket to make this less likely or claim
2320 * the memory directly.
2321 */
2322 if (cache->uc_allocbucket == NULL)
2323 cache->uc_allocbucket = bucket;
2324 else
2325 LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
2326 ZONE_UNLOCK(zone);
2327 goto zalloc_start;
2328 }
2329
2330 /*
2331 * We may not be able to get a bucket so return an actual item.
2332 */
2333 #ifdef UMA_DEBUG
2334 printf("uma_zalloc_arg: Bucketzone returned NULL\n");
2335 #endif
2336
2337 zalloc_item:
2338 item = zone_alloc_item(zone, udata, flags);
2339
2340 return (item);
2341 }
2342
2343 static uma_slab_t
keg_fetch_slab(uma_keg_t keg,uma_zone_t zone,int flags)2344 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
2345 {
2346 uma_slab_t slab;
2347 int reserve;
2348
2349 mtx_assert(&keg->uk_lock, MA_OWNED);
2350 slab = NULL;
2351 reserve = 0;
2352 if ((flags & M_USE_RESERVE) == 0)
2353 reserve = keg->uk_reserve;
2354
2355 for (;;) {
2356 /*
2357 * Find a slab with some space. Prefer slabs that are partially
2358 * used over those that are totally full. This helps to reduce
2359 * fragmentation.
2360 */
2361 if (keg->uk_free > reserve) {
2362 if (!LIST_EMPTY(&keg->uk_part_slab)) {
2363 slab = LIST_FIRST(&keg->uk_part_slab);
2364 } else {
2365 slab = LIST_FIRST(&keg->uk_free_slab);
2366 LIST_REMOVE(slab, us_link);
2367 LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
2368 us_link);
2369 }
2370 MPASS(slab->us_keg == keg);
2371 return (slab);
2372 }
2373
2374 /*
2375 * M_NOVM means don't ask at all!
2376 */
2377 if (flags & M_NOVM)
2378 break;
2379
2380 if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
2381 keg->uk_flags |= UMA_ZFLAG_FULL;
2382 /*
2383 * If this is not a multi-zone, set the FULL bit.
2384 * Otherwise slab_multi() takes care of it.
2385 */
2386 if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
2387 zone->uz_flags |= UMA_ZFLAG_FULL;
2388 zone_log_warning(zone);
2389 zone_maxaction(zone);
2390 }
2391 if (flags & M_NOWAIT)
2392 break;
2393 zone->uz_sleeps++;
2394 msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
2395 continue;
2396 }
2397 slab = keg_alloc_slab(keg, zone, flags);
2398 /*
2399 * If we got a slab here it's safe to mark it partially used
2400 * and return. We assume that the caller is going to remove
2401 * at least one item.
2402 */
2403 if (slab) {
2404 MPASS(slab->us_keg == keg);
2405 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2406 return (slab);
2407 }
2408 /*
2409 * We might not have been able to get a slab but another cpu
2410 * could have while we were unlocked. Check again before we
2411 * fail.
2412 */
2413 flags |= M_NOVM;
2414 }
2415 return (slab);
2416 }
2417
2418 static uma_slab_t
zone_fetch_slab(uma_zone_t zone,uma_keg_t keg,int flags)2419 zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2420 {
2421 uma_slab_t slab;
2422
2423 if (keg == NULL) {
2424 keg = zone_first_keg(zone);
2425 KEG_LOCK(keg);
2426 }
2427
2428 for (;;) {
2429 slab = keg_fetch_slab(keg, zone, flags);
2430 if (slab)
2431 return (slab);
2432 if (flags & (M_NOWAIT | M_NOVM))
2433 break;
2434 }
2435 KEG_UNLOCK(keg);
2436 return (NULL);
2437 }
2438
2439 /*
2440 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
2441 * with the keg locked. On NULL no lock is held.
2442 *
2443 * The last pointer is used to seed the search. It is not required.
2444 */
2445 static uma_slab_t
zone_fetch_slab_multi(uma_zone_t zone,uma_keg_t last,int rflags)2446 zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2447 {
2448 uma_klink_t klink;
2449 uma_slab_t slab;
2450 uma_keg_t keg;
2451 int flags;
2452 int empty;
2453 int full;
2454
2455 /*
2456 * Don't wait on the first pass. This will skip limit tests
2457 * as well. We don't want to block if we can find a provider
2458 * without blocking.
2459 */
2460 flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2461 /*
2462 * Use the last slab allocated as a hint for where to start
2463 * the search.
2464 */
2465 if (last != NULL) {
2466 slab = keg_fetch_slab(last, zone, flags);
2467 if (slab)
2468 return (slab);
2469 KEG_UNLOCK(last);
2470 }
2471 /*
2472 * Loop until we have a slab incase of transient failures
2473 * while M_WAITOK is specified. I'm not sure this is 100%
2474 * required but we've done it for so long now.
2475 */
2476 for (;;) {
2477 empty = 0;
2478 full = 0;
2479 /*
2480 * Search the available kegs for slabs. Be careful to hold the
2481 * correct lock while calling into the keg layer.
2482 */
2483 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2484 keg = klink->kl_keg;
2485 KEG_LOCK(keg);
2486 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2487 slab = keg_fetch_slab(keg, zone, flags);
2488 if (slab)
2489 return (slab);
2490 }
2491 if (keg->uk_flags & UMA_ZFLAG_FULL)
2492 full++;
2493 else
2494 empty++;
2495 KEG_UNLOCK(keg);
2496 }
2497 if (rflags & (M_NOWAIT | M_NOVM))
2498 break;
2499 flags = rflags;
2500 /*
2501 * All kegs are full. XXX We can't atomically check all kegs
2502 * and sleep so just sleep for a short period and retry.
2503 */
2504 if (full && !empty) {
2505 ZONE_LOCK(zone);
2506 zone->uz_flags |= UMA_ZFLAG_FULL;
2507 zone->uz_sleeps++;
2508 zone_log_warning(zone);
2509 zone_maxaction(zone);
2510 msleep(zone, zone->uz_lockptr, PVM,
2511 "zonelimit", hz/100);
2512 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2513 ZONE_UNLOCK(zone);
2514 continue;
2515 }
2516 }
2517 return (NULL);
2518 }
2519
2520 static void *
slab_alloc_item(uma_keg_t keg,uma_slab_t slab)2521 slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2522 {
2523 void *item;
2524 uint8_t freei;
2525
2526 MPASS(keg == slab->us_keg);
2527 mtx_assert(&keg->uk_lock, MA_OWNED);
2528
2529 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2530 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2531 item = slab->us_data + (keg->uk_rsize * freei);
2532 slab->us_freecount--;
2533 keg->uk_free--;
2534
2535 /* Move this slab to the full list */
2536 if (slab->us_freecount == 0) {
2537 LIST_REMOVE(slab, us_link);
2538 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2539 }
2540
2541 return (item);
2542 }
2543
2544 static int
zone_import(uma_zone_t zone,void ** bucket,int max,int flags)2545 zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2546 {
2547 uma_slab_t slab;
2548 uma_keg_t keg;
2549 int i;
2550
2551 slab = NULL;
2552 keg = NULL;
2553 /* Try to keep the buckets totally full */
2554 for (i = 0; i < max; ) {
2555 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2556 break;
2557 keg = slab->us_keg;
2558 while (slab->us_freecount && i < max) {
2559 bucket[i++] = slab_alloc_item(keg, slab);
2560 if (keg->uk_free <= keg->uk_reserve)
2561 break;
2562 }
2563 /* Don't grab more than one slab at a time. */
2564 flags &= ~M_WAITOK;
2565 flags |= M_NOWAIT;
2566 }
2567 if (slab != NULL)
2568 KEG_UNLOCK(keg);
2569
2570 return i;
2571 }
2572
2573 static uma_bucket_t
zone_alloc_bucket(uma_zone_t zone,void * udata,int flags)2574 zone_alloc_bucket(uma_zone_t zone, void *udata, int flags)
2575 {
2576 uma_bucket_t bucket;
2577 int max;
2578
2579 /* Don't wait for buckets, preserve caller's NOVM setting. */
2580 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
2581 if (bucket == NULL)
2582 return (NULL);
2583
2584 max = MIN(bucket->ub_entries, zone->uz_count);
2585 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2586 max, flags);
2587
2588 /*
2589 * Initialize the memory if necessary.
2590 */
2591 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2592 int i;
2593
2594 for (i = 0; i < bucket->ub_cnt; i++)
2595 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
2596 flags) != 0)
2597 break;
2598 /*
2599 * If we couldn't initialize the whole bucket, put the
2600 * rest back onto the freelist.
2601 */
2602 if (i != bucket->ub_cnt) {
2603 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
2604 bucket->ub_cnt - i);
2605 #ifdef INVARIANTS
2606 bzero(&bucket->ub_bucket[i],
2607 sizeof(void *) * (bucket->ub_cnt - i));
2608 #endif
2609 bucket->ub_cnt = i;
2610 }
2611 }
2612
2613 if (bucket->ub_cnt == 0) {
2614 bucket_free(zone, bucket, udata);
2615 atomic_add_long(&zone->uz_fails, 1);
2616 return (NULL);
2617 }
2618
2619 return (bucket);
2620 }
2621
2622 /*
2623 * Allocates a single item from a zone.
2624 *
2625 * Arguments
2626 * zone The zone to alloc for.
2627 * udata The data to be passed to the constructor.
2628 * flags M_WAITOK, M_NOWAIT, M_ZERO.
2629 *
2630 * Returns
2631 * NULL if there is no memory and M_NOWAIT is set
2632 * An item if successful
2633 */
2634
2635 static void *
zone_alloc_item(uma_zone_t zone,void * udata,int flags)2636 zone_alloc_item(uma_zone_t zone, void *udata, int flags)
2637 {
2638 void *item;
2639
2640 item = NULL;
2641
2642 #ifdef UMA_DEBUG_ALLOC
2643 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2644 #endif
2645 if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
2646 goto fail;
2647 atomic_add_long(&zone->uz_allocs, 1);
2648
2649 /*
2650 * We have to call both the zone's init (not the keg's init)
2651 * and the zone's ctor. This is because the item is going from
2652 * a keg slab directly to the user, and the user is expecting it
2653 * to be both zone-init'd as well as zone-ctor'd.
2654 */
2655 if (zone->uz_init != NULL) {
2656 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
2657 zone_free_item(zone, item, udata, SKIP_FINI);
2658 goto fail;
2659 }
2660 }
2661 if (zone->uz_ctor != NULL) {
2662 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2663 zone_free_item(zone, item, udata, SKIP_DTOR);
2664 goto fail;
2665 }
2666 }
2667 #ifdef INVARIANTS
2668 uma_dbg_alloc(zone, NULL, item);
2669 #endif
2670 if (flags & M_ZERO)
2671 uma_zero_item(item, zone);
2672
2673 return (item);
2674
2675 fail:
2676 atomic_add_long(&zone->uz_fails, 1);
2677 return (NULL);
2678 }
2679
2680 /* See uma.h */
2681 void
uma_zfree_arg(uma_zone_t zone,void * item,void * udata)2682 uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2683 {
2684 uma_cache_t cache;
2685 uma_bucket_t bucket;
2686 int lockfail;
2687 int cpu;
2688
2689 #ifdef UMA_DEBUG_ALLOC_1
2690 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
2691 #endif
2692 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
2693 zone->uz_name);
2694
2695 KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
2696 ("uma_zfree_arg: called with spinlock or critical section held"));
2697
2698 /* uma_zfree(..., NULL) does nothing, to match free(9). */
2699 if (item == NULL)
2700 return;
2701 #ifdef DEBUG_MEMGUARD
2702 if (is_memguard_addr(item)) {
2703 if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
2704 zone->uz_dtor(item, zone->uz_size, udata);
2705 if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
2706 zone->uz_fini(item, zone->uz_size);
2707 memguard_free(item);
2708 return;
2709 }
2710 #endif
2711 #ifdef INVARIANTS
2712 if (zone->uz_flags & UMA_ZONE_MALLOC)
2713 uma_dbg_free(zone, udata, item);
2714 else
2715 uma_dbg_free(zone, NULL, item);
2716 #endif
2717 if (zone->uz_dtor != NULL)
2718 zone->uz_dtor(item, zone->uz_size, udata);
2719
2720 /*
2721 * The race here is acceptable. If we miss it we'll just have to wait
2722 * a little longer for the limits to be reset.
2723 */
2724 if (zone->uz_flags & UMA_ZFLAG_FULL)
2725 goto zfree_item;
2726
2727 /*
2728 * If possible, free to the per-CPU cache. There are two
2729 * requirements for safe access to the per-CPU cache: (1) the thread
2730 * accessing the cache must not be preempted or yield during access,
2731 * and (2) the thread must not migrate CPUs without switching which
2732 * cache it accesses. We rely on a critical section to prevent
2733 * preemption and migration. We release the critical section in
2734 * order to acquire the zone mutex if we are unable to free to the
2735 * current cache; when we re-acquire the critical section, we must
2736 * detect and handle migration if it has occurred.
2737 */
2738 zfree_restart:
2739 critical_enter();
2740 cpu = curcpu;
2741 cache = &zone->uz_cpu[cpu];
2742
2743 zfree_start:
2744 /*
2745 * Try to free into the allocbucket first to give LIFO ordering
2746 * for cache-hot datastructures. Spill over into the freebucket
2747 * if necessary. Alloc will swap them if one runs dry.
2748 */
2749 bucket = cache->uc_allocbucket;
2750 if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries)
2751 bucket = cache->uc_freebucket;
2752 if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
2753 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
2754 ("uma_zfree: Freeing to non free bucket index."));
2755 bucket->ub_bucket[bucket->ub_cnt] = item;
2756 bucket->ub_cnt++;
2757 cache->uc_frees++;
2758 critical_exit();
2759 return;
2760 }
2761
2762 /*
2763 * We must go back the zone, which requires acquiring the zone lock,
2764 * which in turn means we must release and re-acquire the critical
2765 * section. Since the critical section is released, we may be
2766 * preempted or migrate. As such, make sure not to maintain any
2767 * thread-local state specific to the cache from prior to releasing
2768 * the critical section.
2769 */
2770 critical_exit();
2771 if (zone->uz_count == 0 || bucketdisable)
2772 goto zfree_item;
2773
2774 lockfail = 0;
2775 if (ZONE_TRYLOCK(zone) == 0) {
2776 /* Record contention to size the buckets. */
2777 ZONE_LOCK(zone);
2778 lockfail = 1;
2779 }
2780 critical_enter();
2781 cpu = curcpu;
2782 cache = &zone->uz_cpu[cpu];
2783
2784 /*
2785 * Since we have locked the zone we may as well send back our stats.
2786 */
2787 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2788 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2789 cache->uc_allocs = 0;
2790 cache->uc_frees = 0;
2791
2792 bucket = cache->uc_freebucket;
2793 if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
2794 ZONE_UNLOCK(zone);
2795 goto zfree_start;
2796 }
2797 cache->uc_freebucket = NULL;
2798
2799 /* Can we throw this on the zone full list? */
2800 if (bucket != NULL) {
2801 #ifdef UMA_DEBUG_ALLOC
2802 printf("uma_zfree: Putting old bucket on the free list.\n");
2803 #endif
2804 /* ub_cnt is pointing to the last free item */
2805 KASSERT(bucket->ub_cnt != 0,
2806 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
2807 LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
2808 }
2809
2810 /* We are no longer associated with this CPU. */
2811 critical_exit();
2812
2813 /*
2814 * We bump the uz count when the cache size is insufficient to
2815 * handle the working set.
2816 */
2817 if (lockfail && zone->uz_count < BUCKET_MAX)
2818 zone->uz_count++;
2819 ZONE_UNLOCK(zone);
2820
2821 #ifdef UMA_DEBUG_ALLOC
2822 printf("uma_zfree: Allocating new free bucket.\n");
2823 #endif
2824 bucket = bucket_alloc(zone, udata, M_NOWAIT);
2825 if (bucket) {
2826 critical_enter();
2827 cpu = curcpu;
2828 cache = &zone->uz_cpu[cpu];
2829 if (cache->uc_freebucket == NULL) {
2830 cache->uc_freebucket = bucket;
2831 goto zfree_start;
2832 }
2833 /*
2834 * We lost the race, start over. We have to drop our
2835 * critical section to free the bucket.
2836 */
2837 critical_exit();
2838 bucket_free(zone, bucket, udata);
2839 goto zfree_restart;
2840 }
2841
2842 /*
2843 * If nothing else caught this, we'll just do an internal free.
2844 */
2845 zfree_item:
2846 zone_free_item(zone, item, udata, SKIP_DTOR);
2847
2848 return;
2849 }
2850
2851 static void
slab_free_item(uma_keg_t keg,uma_slab_t slab,void * item)2852 slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
2853 {
2854 uint8_t freei;
2855
2856 mtx_assert(&keg->uk_lock, MA_OWNED);
2857 MPASS(keg == slab->us_keg);
2858
2859 /* Do we need to remove from any lists? */
2860 if (slab->us_freecount+1 == keg->uk_ipers) {
2861 LIST_REMOVE(slab, us_link);
2862 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2863 } else if (slab->us_freecount == 0) {
2864 LIST_REMOVE(slab, us_link);
2865 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2866 }
2867
2868 /* Slab management. */
2869 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2870 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
2871 slab->us_freecount++;
2872
2873 /* Keg statistics. */
2874 keg->uk_free++;
2875 }
2876
2877 static void
zone_release(uma_zone_t zone,void ** bucket,int cnt)2878 zone_release(uma_zone_t zone, void **bucket, int cnt)
2879 {
2880 void *item;
2881 uma_slab_t slab;
2882 uma_keg_t keg;
2883 uint8_t *mem;
2884 int clearfull;
2885 int i;
2886
2887 clearfull = 0;
2888 keg = zone_first_keg(zone);
2889 KEG_LOCK(keg);
2890 for (i = 0; i < cnt; i++) {
2891 item = bucket[i];
2892 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2893 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2894 if (zone->uz_flags & UMA_ZONE_HASH) {
2895 slab = hash_sfind(&keg->uk_hash, mem);
2896 } else {
2897 mem += keg->uk_pgoff;
2898 slab = (uma_slab_t)mem;
2899 }
2900 } else {
2901 slab = vtoslab((vm_offset_t)item);
2902 if (slab->us_keg != keg) {
2903 KEG_UNLOCK(keg);
2904 keg = slab->us_keg;
2905 KEG_LOCK(keg);
2906 }
2907 }
2908 slab_free_item(keg, slab, item);
2909 if (keg->uk_flags & UMA_ZFLAG_FULL) {
2910 if (keg->uk_pages < keg->uk_maxpages) {
2911 keg->uk_flags &= ~UMA_ZFLAG_FULL;
2912 clearfull = 1;
2913 }
2914
2915 /*
2916 * We can handle one more allocation. Since we're
2917 * clearing ZFLAG_FULL, wake up all procs blocked
2918 * on pages. This should be uncommon, so keeping this
2919 * simple for now (rather than adding count of blocked
2920 * threads etc).
2921 */
2922 wakeup(keg);
2923 }
2924 }
2925 KEG_UNLOCK(keg);
2926 if (clearfull) {
2927 ZONE_LOCK(zone);
2928 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2929 wakeup(zone);
2930 ZONE_UNLOCK(zone);
2931 }
2932
2933 }
2934
2935 /*
2936 * Frees a single item to any zone.
2937 *
2938 * Arguments:
2939 * zone The zone to free to
2940 * item The item we're freeing
2941 * udata User supplied data for the dtor
2942 * skip Skip dtors and finis
2943 */
2944 static void
zone_free_item(uma_zone_t zone,void * item,void * udata,enum zfreeskip skip)2945 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
2946 {
2947
2948 #ifdef INVARIANTS
2949 if (skip == SKIP_NONE) {
2950 if (zone->uz_flags & UMA_ZONE_MALLOC)
2951 uma_dbg_free(zone, udata, item);
2952 else
2953 uma_dbg_free(zone, NULL, item);
2954 }
2955 #endif
2956 if (skip < SKIP_DTOR && zone->uz_dtor)
2957 zone->uz_dtor(item, zone->uz_size, udata);
2958
2959 if (skip < SKIP_FINI && zone->uz_fini)
2960 zone->uz_fini(item, zone->uz_size);
2961
2962 atomic_add_long(&zone->uz_frees, 1);
2963 zone->uz_release(zone->uz_arg, &item, 1);
2964 }
2965
2966 /* See uma.h */
2967 int
uma_zone_set_max(uma_zone_t zone,int nitems)2968 uma_zone_set_max(uma_zone_t zone, int nitems)
2969 {
2970 uma_keg_t keg;
2971
2972 keg = zone_first_keg(zone);
2973 if (keg == NULL)
2974 return (0);
2975 KEG_LOCK(keg);
2976 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2977 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2978 keg->uk_maxpages += keg->uk_ppera;
2979 nitems = keg->uk_maxpages * keg->uk_ipers;
2980 KEG_UNLOCK(keg);
2981
2982 return (nitems);
2983 }
2984
2985 /* See uma.h */
2986 int
uma_zone_get_max(uma_zone_t zone)2987 uma_zone_get_max(uma_zone_t zone)
2988 {
2989 int nitems;
2990 uma_keg_t keg;
2991
2992 keg = zone_first_keg(zone);
2993 if (keg == NULL)
2994 return (0);
2995 KEG_LOCK(keg);
2996 nitems = keg->uk_maxpages * keg->uk_ipers;
2997 KEG_UNLOCK(keg);
2998
2999 return (nitems);
3000 }
3001
3002 /* See uma.h */
3003 void
uma_zone_set_warning(uma_zone_t zone,const char * warning)3004 uma_zone_set_warning(uma_zone_t zone, const char *warning)
3005 {
3006
3007 ZONE_LOCK(zone);
3008 zone->uz_warning = warning;
3009 ZONE_UNLOCK(zone);
3010 }
3011
3012 /* See uma.h */
3013 void
uma_zone_set_maxaction(uma_zone_t zone,uma_maxaction_t maxaction)3014 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
3015 {
3016
3017 ZONE_LOCK(zone);
3018 zone->uz_maxaction = maxaction;
3019 ZONE_UNLOCK(zone);
3020 }
3021
3022 /* See uma.h */
3023 int
uma_zone_get_cur(uma_zone_t zone)3024 uma_zone_get_cur(uma_zone_t zone)
3025 {
3026 int64_t nitems;
3027 u_int i;
3028
3029 ZONE_LOCK(zone);
3030 nitems = zone->uz_allocs - zone->uz_frees;
3031 CPU_FOREACH(i) {
3032 /*
3033 * See the comment in sysctl_vm_zone_stats() regarding the
3034 * safety of accessing the per-cpu caches. With the zone lock
3035 * held, it is safe, but can potentially result in stale data.
3036 */
3037 nitems += zone->uz_cpu[i].uc_allocs -
3038 zone->uz_cpu[i].uc_frees;
3039 }
3040 ZONE_UNLOCK(zone);
3041
3042 return (nitems < 0 ? 0 : nitems);
3043 }
3044
3045 /* See uma.h */
3046 void
uma_zone_set_init(uma_zone_t zone,uma_init uminit)3047 uma_zone_set_init(uma_zone_t zone, uma_init uminit)
3048 {
3049 uma_keg_t keg;
3050
3051 keg = zone_first_keg(zone);
3052 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
3053 KEG_LOCK(keg);
3054 KASSERT(keg->uk_pages == 0,
3055 ("uma_zone_set_init on non-empty keg"));
3056 keg->uk_init = uminit;
3057 KEG_UNLOCK(keg);
3058 }
3059
3060 /* See uma.h */
3061 void
uma_zone_set_fini(uma_zone_t zone,uma_fini fini)3062 uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
3063 {
3064 uma_keg_t keg;
3065
3066 keg = zone_first_keg(zone);
3067 KASSERT(keg != NULL, ("uma_zone_set_fini: Invalid zone type"));
3068 KEG_LOCK(keg);
3069 KASSERT(keg->uk_pages == 0,
3070 ("uma_zone_set_fini on non-empty keg"));
3071 keg->uk_fini = fini;
3072 KEG_UNLOCK(keg);
3073 }
3074
3075 /* See uma.h */
3076 void
uma_zone_set_zinit(uma_zone_t zone,uma_init zinit)3077 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
3078 {
3079
3080 ZONE_LOCK(zone);
3081 KASSERT(zone_first_keg(zone)->uk_pages == 0,
3082 ("uma_zone_set_zinit on non-empty keg"));
3083 zone->uz_init = zinit;
3084 ZONE_UNLOCK(zone);
3085 }
3086
3087 /* See uma.h */
3088 void
uma_zone_set_zfini(uma_zone_t zone,uma_fini zfini)3089 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
3090 {
3091
3092 ZONE_LOCK(zone);
3093 KASSERT(zone_first_keg(zone)->uk_pages == 0,
3094 ("uma_zone_set_zfini on non-empty keg"));
3095 zone->uz_fini = zfini;
3096 ZONE_UNLOCK(zone);
3097 }
3098
3099 /* See uma.h */
3100 /* XXX uk_freef is not actually used with the zone locked */
3101 void
uma_zone_set_freef(uma_zone_t zone,uma_free freef)3102 uma_zone_set_freef(uma_zone_t zone, uma_free freef)
3103 {
3104 uma_keg_t keg;
3105
3106 keg = zone_first_keg(zone);
3107 KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type"));
3108 KEG_LOCK(keg);
3109 keg->uk_freef = freef;
3110 KEG_UNLOCK(keg);
3111 }
3112
3113 /* See uma.h */
3114 /* XXX uk_allocf is not actually used with the zone locked */
3115 void
uma_zone_set_allocf(uma_zone_t zone,uma_alloc allocf)3116 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
3117 {
3118 uma_keg_t keg;
3119
3120 keg = zone_first_keg(zone);
3121 KEG_LOCK(keg);
3122 keg->uk_allocf = allocf;
3123 KEG_UNLOCK(keg);
3124 }
3125
3126 /* See uma.h */
3127 void
uma_zone_reserve(uma_zone_t zone,int items)3128 uma_zone_reserve(uma_zone_t zone, int items)
3129 {
3130 uma_keg_t keg;
3131
3132 keg = zone_first_keg(zone);
3133 if (keg == NULL)
3134 return;
3135 KEG_LOCK(keg);
3136 keg->uk_reserve = items;
3137 KEG_UNLOCK(keg);
3138
3139 return;
3140 }
3141
3142 /* See uma.h */
3143 int
uma_zone_reserve_kva(uma_zone_t zone,int count)3144 uma_zone_reserve_kva(uma_zone_t zone, int count)
3145 {
3146 uma_keg_t keg;
3147 vm_offset_t kva;
3148 u_int pages;
3149
3150 keg = zone_first_keg(zone);
3151 if (keg == NULL)
3152 return (0);
3153 pages = count / keg->uk_ipers;
3154
3155 if (pages * keg->uk_ipers < count)
3156 pages++;
3157
3158 #ifdef UMA_MD_SMALL_ALLOC
3159 if (keg->uk_ppera > 1) {
3160 #else
3161 if (1) {
3162 #endif
3163 kva = kva_alloc((vm_size_t)pages * UMA_SLAB_SIZE);
3164 if (kva == 0)
3165 return (0);
3166 } else
3167 kva = 0;
3168 KEG_LOCK(keg);
3169 keg->uk_kva = kva;
3170 keg->uk_offset = 0;
3171 keg->uk_maxpages = pages;
3172 #ifdef UMA_MD_SMALL_ALLOC
3173 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
3174 #else
3175 keg->uk_allocf = noobj_alloc;
3176 #endif
3177 keg->uk_flags |= UMA_ZONE_NOFREE;
3178 KEG_UNLOCK(keg);
3179
3180 return (1);
3181 }
3182
3183 /* See uma.h */
3184 void
3185 uma_prealloc(uma_zone_t zone, int items)
3186 {
3187 int slabs;
3188 uma_slab_t slab;
3189 uma_keg_t keg;
3190
3191 keg = zone_first_keg(zone);
3192 if (keg == NULL)
3193 return;
3194 KEG_LOCK(keg);
3195 slabs = items / keg->uk_ipers;
3196 if (slabs * keg->uk_ipers < items)
3197 slabs++;
3198 while (slabs > 0) {
3199 slab = keg_alloc_slab(keg, zone, M_WAITOK);
3200 if (slab == NULL)
3201 break;
3202 MPASS(slab->us_keg == keg);
3203 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
3204 slabs--;
3205 }
3206 KEG_UNLOCK(keg);
3207 }
3208
3209 /* See uma.h */
3210 uint32_t *
3211 uma_find_refcnt(uma_zone_t zone, void *item)
3212 {
3213 uma_slabrefcnt_t slabref;
3214 uma_slab_t slab;
3215 uma_keg_t keg;
3216 uint32_t *refcnt;
3217 int idx;
3218
3219 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
3220 slabref = (uma_slabrefcnt_t)slab;
3221 keg = slab->us_keg;
3222 KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
3223 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3224 idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
3225 refcnt = &slabref->us_refcnt[idx];
3226 return refcnt;
3227 }
3228
3229 /* See uma.h */
3230 static void
3231 uma_reclaim_locked(bool kmem_danger)
3232 {
3233
3234 #ifdef UMA_DEBUG
3235 printf("UMA: vm asked us to release pages!\n");
3236 #endif
3237 sx_assert(&uma_drain_lock, SA_XLOCKED);
3238 bucket_enable();
3239 zone_foreach(zone_drain);
3240 if (vm_page_count_min() || kmem_danger) {
3241 cache_drain_safe(NULL);
3242 zone_foreach(zone_drain);
3243 }
3244 /*
3245 * Some slabs may have been freed but this zone will be visited early
3246 * we visit again so that we can free pages that are empty once other
3247 * zones are drained. We have to do the same for buckets.
3248 */
3249 zone_drain(slabzone);
3250 zone_drain(slabrefzone);
3251 bucket_zone_drain();
3252 }
3253
3254 void
3255 uma_reclaim(void)
3256 {
3257
3258 sx_xlock(&uma_drain_lock);
3259 uma_reclaim_locked(false);
3260 sx_xunlock(&uma_drain_lock);
3261 }
3262
3263 static int uma_reclaim_needed;
3264
3265 void
3266 uma_reclaim_wakeup(void)
3267 {
3268
3269 uma_reclaim_needed = 1;
3270 wakeup(&uma_reclaim_needed);
3271 }
3272
3273 void
3274 uma_reclaim_worker(void *arg __unused)
3275 {
3276
3277 sx_xlock(&uma_drain_lock);
3278 for (;;) {
3279 sx_sleep(&uma_reclaim_needed, &uma_drain_lock, PVM,
3280 "umarcl", 0);
3281 if (uma_reclaim_needed) {
3282 uma_reclaim_needed = 0;
3283 uma_reclaim_locked(true);
3284 }
3285 }
3286 }
3287
3288 /* See uma.h */
3289 int
3290 uma_zone_exhausted(uma_zone_t zone)
3291 {
3292 int full;
3293
3294 ZONE_LOCK(zone);
3295 full = (zone->uz_flags & UMA_ZFLAG_FULL);
3296 ZONE_UNLOCK(zone);
3297 return (full);
3298 }
3299
3300 int
3301 uma_zone_exhausted_nolock(uma_zone_t zone)
3302 {
3303 return (zone->uz_flags & UMA_ZFLAG_FULL);
3304 }
3305
3306 void *
3307 uma_large_malloc(vm_size_t size, int wait)
3308 {
3309 void *mem;
3310 uma_slab_t slab;
3311 uint8_t flags;
3312
3313 slab = zone_alloc_item(slabzone, NULL, wait);
3314 if (slab == NULL)
3315 return (NULL);
3316 mem = page_alloc(NULL, size, &flags, wait);
3317 if (mem) {
3318 vsetslab((vm_offset_t)mem, slab);
3319 slab->us_data = mem;
3320 slab->us_flags = flags | UMA_SLAB_MALLOC;
3321 slab->us_size = size;
3322 } else {
3323 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3324 }
3325
3326 return (mem);
3327 }
3328
3329 void
3330 uma_large_free(uma_slab_t slab)
3331 {
3332
3333 page_free(slab->us_data, slab->us_size, slab->us_flags);
3334 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3335 }
3336
3337 static void
3338 uma_zero_item(void *item, uma_zone_t zone)
3339 {
3340
3341 if (zone->uz_flags & UMA_ZONE_PCPU) {
3342 for (int i = 0; i < mp_ncpus; i++)
3343 bzero(zpcpu_get_cpu(item, i), zone->uz_size);
3344 } else
3345 bzero(item, zone->uz_size);
3346 }
3347
3348 void
3349 uma_print_stats(void)
3350 {
3351 zone_foreach(uma_print_zone);
3352 }
3353
3354 static void
3355 slab_print(uma_slab_t slab)
3356 {
3357 printf("slab: keg %p, data %p, freecount %d\n",
3358 slab->us_keg, slab->us_data, slab->us_freecount);
3359 }
3360
3361 static void
3362 cache_print(uma_cache_t cache)
3363 {
3364 printf("alloc: %p(%d), free: %p(%d)\n",
3365 cache->uc_allocbucket,
3366 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
3367 cache->uc_freebucket,
3368 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
3369 }
3370
3371 static void
3372 uma_print_keg(uma_keg_t keg)
3373 {
3374 uma_slab_t slab;
3375
3376 printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
3377 "out %d free %d limit %d\n",
3378 keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
3379 keg->uk_ipers, keg->uk_ppera,
3380 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free,
3381 (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
3382 printf("Part slabs:\n");
3383 LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
3384 slab_print(slab);
3385 printf("Free slabs:\n");
3386 LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
3387 slab_print(slab);
3388 printf("Full slabs:\n");
3389 LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
3390 slab_print(slab);
3391 }
3392
3393 void
3394 uma_print_zone(uma_zone_t zone)
3395 {
3396 uma_cache_t cache;
3397 uma_klink_t kl;
3398 int i;
3399
3400 printf("zone: %s(%p) size %d flags %#x\n",
3401 zone->uz_name, zone, zone->uz_size, zone->uz_flags);
3402 LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
3403 uma_print_keg(kl->kl_keg);
3404 CPU_FOREACH(i) {
3405 cache = &zone->uz_cpu[i];
3406 printf("CPU %d Cache:\n", i);
3407 cache_print(cache);
3408 }
3409 }
3410
3411 #ifdef DDB
3412 /*
3413 * Generate statistics across both the zone and its per-cpu cache's. Return
3414 * desired statistics if the pointer is non-NULL for that statistic.
3415 *
3416 * Note: does not update the zone statistics, as it can't safely clear the
3417 * per-CPU cache statistic.
3418 *
3419 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
3420 * safe from off-CPU; we should modify the caches to track this information
3421 * directly so that we don't have to.
3422 */
3423 static void
3424 uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
3425 uint64_t *freesp, uint64_t *sleepsp)
3426 {
3427 uma_cache_t cache;
3428 uint64_t allocs, frees, sleeps;
3429 int cachefree, cpu;
3430
3431 allocs = frees = sleeps = 0;
3432 cachefree = 0;
3433 CPU_FOREACH(cpu) {
3434 cache = &z->uz_cpu[cpu];
3435 if (cache->uc_allocbucket != NULL)
3436 cachefree += cache->uc_allocbucket->ub_cnt;
3437 if (cache->uc_freebucket != NULL)
3438 cachefree += cache->uc_freebucket->ub_cnt;
3439 allocs += cache->uc_allocs;
3440 frees += cache->uc_frees;
3441 }
3442 allocs += z->uz_allocs;
3443 frees += z->uz_frees;
3444 sleeps += z->uz_sleeps;
3445 if (cachefreep != NULL)
3446 *cachefreep = cachefree;
3447 if (allocsp != NULL)
3448 *allocsp = allocs;
3449 if (freesp != NULL)
3450 *freesp = frees;
3451 if (sleepsp != NULL)
3452 *sleepsp = sleeps;
3453 }
3454 #endif /* DDB */
3455
3456 static int
3457 sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
3458 {
3459 uma_keg_t kz;
3460 uma_zone_t z;
3461 int count;
3462
3463 count = 0;
3464 rw_rlock(&uma_rwlock);
3465 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3466 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3467 count++;
3468 }
3469 rw_runlock(&uma_rwlock);
3470 return (sysctl_handle_int(oidp, &count, 0, req));
3471 }
3472
3473 static int
3474 sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
3475 {
3476 struct uma_stream_header ush;
3477 struct uma_type_header uth;
3478 struct uma_percpu_stat ups;
3479 uma_bucket_t bucket;
3480 struct sbuf sbuf;
3481 uma_cache_t cache;
3482 uma_klink_t kl;
3483 uma_keg_t kz;
3484 uma_zone_t z;
3485 uma_keg_t k;
3486 int count, error, i;
3487
3488 error = sysctl_wire_old_buffer(req, 0);
3489 if (error != 0)
3490 return (error);
3491 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
3492 sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL);
3493
3494 count = 0;
3495 rw_rlock(&uma_rwlock);
3496 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3497 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3498 count++;
3499 }
3500
3501 /*
3502 * Insert stream header.
3503 */
3504 bzero(&ush, sizeof(ush));
3505 ush.ush_version = UMA_STREAM_VERSION;
3506 ush.ush_maxcpus = (mp_maxid + 1);
3507 ush.ush_count = count;
3508 (void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
3509
3510 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3511 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3512 bzero(&uth, sizeof(uth));
3513 ZONE_LOCK(z);
3514 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
3515 uth.uth_align = kz->uk_align;
3516 uth.uth_size = kz->uk_size;
3517 uth.uth_rsize = kz->uk_rsize;
3518 LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
3519 k = kl->kl_keg;
3520 uth.uth_maxpages += k->uk_maxpages;
3521 uth.uth_pages += k->uk_pages;
3522 uth.uth_keg_free += k->uk_free;
3523 uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
3524 * k->uk_ipers;
3525 }
3526
3527 /*
3528 * A zone is secondary is it is not the first entry
3529 * on the keg's zone list.
3530 */
3531 if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
3532 (LIST_FIRST(&kz->uk_zones) != z))
3533 uth.uth_zone_flags = UTH_ZONE_SECONDARY;
3534
3535 LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
3536 uth.uth_zone_free += bucket->ub_cnt;
3537 uth.uth_allocs = z->uz_allocs;
3538 uth.uth_frees = z->uz_frees;
3539 uth.uth_fails = z->uz_fails;
3540 uth.uth_sleeps = z->uz_sleeps;
3541 (void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
3542 /*
3543 * While it is not normally safe to access the cache
3544 * bucket pointers while not on the CPU that owns the
3545 * cache, we only allow the pointers to be exchanged
3546 * without the zone lock held, not invalidated, so
3547 * accept the possible race associated with bucket
3548 * exchange during monitoring.
3549 */
3550 for (i = 0; i < (mp_maxid + 1); i++) {
3551 bzero(&ups, sizeof(ups));
3552 if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
3553 goto skip;
3554 if (CPU_ABSENT(i))
3555 goto skip;
3556 cache = &z->uz_cpu[i];
3557 if (cache->uc_allocbucket != NULL)
3558 ups.ups_cache_free +=
3559 cache->uc_allocbucket->ub_cnt;
3560 if (cache->uc_freebucket != NULL)
3561 ups.ups_cache_free +=
3562 cache->uc_freebucket->ub_cnt;
3563 ups.ups_allocs = cache->uc_allocs;
3564 ups.ups_frees = cache->uc_frees;
3565 skip:
3566 (void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
3567 }
3568 ZONE_UNLOCK(z);
3569 }
3570 }
3571 rw_runlock(&uma_rwlock);
3572 error = sbuf_finish(&sbuf);
3573 sbuf_delete(&sbuf);
3574 return (error);
3575 }
3576
3577 int
3578 sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
3579 {
3580 uma_zone_t zone = *(uma_zone_t *)arg1;
3581 int error, max;
3582
3583 max = uma_zone_get_max(zone);
3584 error = sysctl_handle_int(oidp, &max, 0, req);
3585 if (error || !req->newptr)
3586 return (error);
3587
3588 uma_zone_set_max(zone, max);
3589
3590 return (0);
3591 }
3592
3593 int
3594 sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
3595 {
3596 uma_zone_t zone = *(uma_zone_t *)arg1;
3597 int cur;
3598
3599 cur = uma_zone_get_cur(zone);
3600 return (sysctl_handle_int(oidp, &cur, 0, req));
3601 }
3602
3603 #ifdef DDB
3604 DB_SHOW_COMMAND(uma, db_show_uma)
3605 {
3606 uint64_t allocs, frees, sleeps;
3607 uma_bucket_t bucket;
3608 uma_keg_t kz;
3609 uma_zone_t z;
3610 int cachefree;
3611
3612 db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used",
3613 "Free", "Requests", "Sleeps", "Bucket");
3614 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3615 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3616 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
3617 allocs = z->uz_allocs;
3618 frees = z->uz_frees;
3619 sleeps = z->uz_sleeps;
3620 cachefree = 0;
3621 } else
3622 uma_zone_sumstat(z, &cachefree, &allocs,
3623 &frees, &sleeps);
3624 if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
3625 (LIST_FIRST(&kz->uk_zones) != z)))
3626 cachefree += kz->uk_free;
3627 LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
3628 cachefree += bucket->ub_cnt;
3629 db_printf("%18s %8ju %8jd %8d %12ju %8ju %8u\n",
3630 z->uz_name, (uintmax_t)kz->uk_size,
3631 (intmax_t)(allocs - frees), cachefree,
3632 (uintmax_t)allocs, sleeps, z->uz_count);
3633 if (db_pager_quit)
3634 return;
3635 }
3636 }
3637 }
3638
3639 DB_SHOW_COMMAND(umacache, db_show_umacache)
3640 {
3641 uint64_t allocs, frees;
3642 uma_bucket_t bucket;
3643 uma_zone_t z;
3644 int cachefree;
3645
3646 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
3647 "Requests", "Bucket");
3648 LIST_FOREACH(z, &uma_cachezones, uz_link) {
3649 uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL);
3650 LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
3651 cachefree += bucket->ub_cnt;
3652 db_printf("%18s %8ju %8jd %8d %12ju %8u\n",
3653 z->uz_name, (uintmax_t)z->uz_size,
3654 (intmax_t)(allocs - frees), cachefree,
3655 (uintmax_t)allocs, z->uz_count);
3656 if (db_pager_quit)
3657 return;
3658 }
3659 }
3660 #endif
3661