323 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
324 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
348 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg);
352 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
353 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
1285 	uma_hash_slab_t slab;  in hash_expand()  local
1302 			slab = LIST_FIRST(&oldhash->uh_slab_hash[idx]);  in hash_expand()
1303 			LIST_REMOVE(slab, uhs_hlink);  in hash_expand()
1304 			hval = UMA_HASH(newhash, slab->uhs_data);  in hash_expand()
1306 			    slab, uhs_hlink);  in hash_expand()
1592 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)  in keg_free_slab()  argument
1600 	    keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera);  in keg_free_slab()
1602 	mem = slab_data(slab, keg);  in keg_free_slab()
1617 		if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) ||  in keg_free_slab()
1620 			keg->uk_fini(slab_item(slab, keg, i), keg->uk_size);  in keg_free_slab()
1622 	flags = slab->us_flags;  in keg_free_slab()
1624 		zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab),  in keg_free_slab()
1636 	uma_slab_t slab, tmp;  in keg_drain_domain()  local
1668 		slab = LIST_FIRST(&dom->ud_free_slab);  in keg_drain_domain()
1669 		LIST_REMOVE(slab, us_link);  in keg_drain_domain()
1670 		LIST_INSERT_HEAD(&freeslabs, slab, us_link);  in keg_drain_domain()
1676 		LIST_FOREACH(slab, &freeslabs, us_link)  in keg_drain_domain()
1677 			UMA_HASH_REMOVE(&keg->uk_hash, slab);  in keg_drain_domain()
1684 	LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp)  in keg_drain_domain()
1685 		keg_free_slab(keg, slab, keg->uk_ipers);  in keg_drain_domain()
1756 	uma_slab_t slab;  in keg_alloc_slab()  local
1767 	slab = NULL;  in keg_alloc_slab()
1775 		slab = &hslab->uhs_slab;  in keg_alloc_slab()
1802 			    slab_tohashslab(slab), NULL, SKIP_NONE);  in keg_alloc_slab()
1816 		slab = (uma_slab_t)(mem + keg->uk_pgoff);  in keg_alloc_slab()
1818 		slab_tohashslab(slab)->uhs_data = mem;  in keg_alloc_slab()
1823 			    zone, slab);  in keg_alloc_slab()
1825 	slab->us_freecount = keg->uk_ipers;  in keg_alloc_slab()
1826 	slab->us_flags = sflags;  in keg_alloc_slab()
1827 	slab->us_domain = domain;  in keg_alloc_slab()
1829 	BIT_FILL(keg->uk_ipers, &slab->us_free);  in keg_alloc_slab()
1831 	BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg));  in keg_alloc_slab()
1836 			if (keg->uk_init(slab_item(slab, keg, i),  in keg_alloc_slab()
1840 			keg_free_slab(keg, slab, i);  in keg_alloc_slab()
1848 	    slab, keg->uk_name, keg);  in keg_alloc_slab()
1851 		UMA_HASH_INSERT(&keg->uk_hash, slab, mem);  in keg_alloc_slab()
1859 	LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in keg_alloc_slab()
1864 	return (slab);  in keg_alloc_slab()
2191 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg)  in slab_dbg_bits()  argument
2194 	return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers)));  in slab_dbg_bits()
3947 	uma_slab_t slab;  in keg_first_slab()  local
3954 	slab = NULL;  in keg_first_slab()
3958 		if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL)  in keg_first_slab()
3959 			return (slab);  in keg_first_slab()
3960 		if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) {  in keg_first_slab()
3961 			LIST_REMOVE(slab, us_link);  in keg_first_slab()
3963 			LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in keg_first_slab()
3964 			return (slab);  in keg_first_slab()
3980 	uma_slab_t slab;  in keg_fetch_free_slab()  local
3990 	    (slab = keg_first_slab(keg, domain, rr)) == NULL) {  in keg_fetch_free_slab()
3994 	return (slab);  in keg_fetch_free_slab()
4001 	uma_slab_t slab;  in keg_fetch_slab()  local
4028 		slab = keg_fetch_free_slab(keg, domain, rr, flags);  in keg_fetch_slab()
4029 		if (slab != NULL)  in keg_fetch_slab()
4030 			return (slab);  in keg_fetch_slab()
4037 			slab = keg_alloc_slab(keg, zone, domain, flags, aflags);  in keg_fetch_slab()
4038 			if (slab != NULL)  in keg_fetch_slab()
4039 				return (slab);  in keg_fetch_slab()
4069 	if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)  in keg_fetch_slab()
4070 		return (slab);  in keg_fetch_slab()
4076 slab_alloc_item(uma_keg_t keg, uma_slab_t slab)  in slab_alloc_item()  argument
4082 	KEG_LOCK_ASSERT(keg, slab->us_domain);  in slab_alloc_item()
4084 	dom = &keg->uk_domain[slab->us_domain];  in slab_alloc_item()
4085 	freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1;  in slab_alloc_item()
4086 	BIT_CLR(keg->uk_ipers, freei, &slab->us_free);  in slab_alloc_item()
4087 	item = slab_item(slab, keg, freei);  in slab_alloc_item()
4088 	slab->us_freecount--;  in slab_alloc_item()
4096 	if (slab->us_freecount == 0) {  in slab_alloc_item()
4097 		LIST_REMOVE(slab, us_link);  in slab_alloc_item()
4098 		LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link);  in slab_alloc_item()
4109 	uma_slab_t slab;  in zone_import()  local
4117 	slab = NULL;  in zone_import()
4121 		if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)  in zone_import()
4126 		dom = &keg->uk_domain[slab->us_domain];  in zone_import()
4128 			bucket[i++] = slab_alloc_item(keg, slab);  in zone_import()
4136 				KEG_UNLOCK(keg, slab->us_domain);  in zone_import()
4152 		} while (slab->us_freecount != 0 && i < max);  in zone_import()
4153 		KEG_UNLOCK(keg, slab->us_domain);  in zone_import()
4832 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)  in slab_free_item()  argument
4839 	KEG_LOCK_ASSERT(keg, slab->us_domain);  in slab_free_item()
4842 	dom = &keg->uk_domain[slab->us_domain];  in slab_free_item()
4843 	if (slab->us_freecount + 1 == keg->uk_ipers) {  in slab_free_item()
4844 		LIST_REMOVE(slab, us_link);  in slab_free_item()
4845 		LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);  in slab_free_item()
4847 	} else if (slab->us_freecount == 0) {  in slab_free_item()
4848 		LIST_REMOVE(slab, us_link);  in slab_free_item()
4849 		LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);  in slab_free_item()
4853 	freei = slab_item_index(slab, keg, item);  in slab_free_item()
4854 	BIT_SET(keg->uk_ipers, freei, &slab->us_free);  in slab_free_item()
4855 	slab->us_freecount++;  in slab_free_item()
4866 	uma_slab_t slab;  in zone_release()  local
4880 			slab = vtoslab((vm_offset_t)item);  in zone_release()
4884 				slab = hash_sfind(&keg->uk_hash, mem);  in zone_release()
4886 				slab = (uma_slab_t)(mem + keg->uk_pgoff);  in zone_release()
4888 		if (lock != KEG_LOCKPTR(keg, slab->us_domain)) {  in zone_release()
4891 			lock = KEG_LOCK(keg, slab->us_domain);  in zone_release()
4893 		slab_free_item(zone, slab, item);  in zone_release()
5240 	uma_slab_t slab;  local
5251 			slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
5253 			if (slab != NULL) {
5254 				dom = &keg->uk_domain[slab->us_domain];
5259 				LIST_REMOVE(slab, us_link);
5260 				LIST_INSERT_HEAD(&dom->ud_free_slab, slab,
5263 				KEG_UNLOCK(keg, slab->us_domain);
5749 	uma_slab_t slab;  local
5767 	slab = hash_sfind(&keg->uk_hash, mem);
5770 	return (slab);
5814 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)  argument
5819 	if (slab == NULL) {
5820 		slab = uma_dbg_getslab(zone, item);
5821 		if (slab == NULL) 
5826 	freei = slab_item_index(slab, keg, item);
5829 	    slab_dbg_bits(slab, keg)))
5831 		    item, zone, zone->uz_name, slab, freei);
5840 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)  argument
5845 	if (slab == NULL) {
5846 		slab = uma_dbg_getslab(zone, item);
5847 		if (slab == NULL) 
5852 	freei = slab_item_index(slab, keg, item);
5856 		    item, zone, zone->uz_name, slab, freei);
5858 	if (slab_item(slab, keg, freei) != item)
5860 		    item, zone, zone->uz_name, slab, freei);
5863 	    slab_dbg_bits(slab, keg)))
5865 		    item, zone, zone->uz_name, slab, freei);