694 	uma_zone_domain_t zdom;  in zone_domain_lock()  local
697 	zdom = ZDOM_GET(zone, domain);  in zone_domain_lock()
699 	if (ZDOM_OWNED(zdom))  in zone_domain_lock()
701 	ZDOM_LOCK(zdom);  in zone_domain_lock()
705 	return (zdom);  in zone_domain_lock()
767 zone_domain_imax_set(uma_zone_domain_t zdom, int nitems)  in zone_domain_imax_set()  argument
771 	old = zdom->uzd_imax;  in zone_domain_imax_set()
775 	} while (atomic_fcmpset_long(&zdom->uzd_imax, &old, nitems) == 0);  in zone_domain_imax_set()
781 	if (zdom->uzd_wss < old - zdom->uzd_bimin)  in zone_domain_imax_set()
782 		zdom->uzd_wss = old - zdom->uzd_bimin;  in zone_domain_imax_set()
783 	zdom->uzd_bimin = nitems;  in zone_domain_imax_set()
791 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim)  in zone_fetch_bucket()  argument
798 	ZDOM_LOCK_ASSERT(zdom);  in zone_fetch_bucket()
800 	if ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) == NULL)  in zone_fetch_bucket()
811 			zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq;  in zone_fetch_bucket()
813 	STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link);  in zone_fetch_bucket()
815 	KASSERT(zdom->uzd_nitems >= bucket->ub_cnt,  in zone_fetch_bucket()
817 	    __func__, zdom->uzd_nitems, bucket->ub_cnt));  in zone_fetch_bucket()
820 	zdom->uzd_nitems -= bucket->ub_cnt;  in zone_fetch_bucket()
827 		cnt = lmin(zdom->uzd_bimin, bucket->ub_cnt);  in zone_fetch_bucket()
828 		atomic_subtract_long(&zdom->uzd_imax, cnt);  in zone_fetch_bucket()
829 		zdom->uzd_bimin -= cnt;  in zone_fetch_bucket()
830 		zdom->uzd_imin -= lmin(zdom->uzd_imin, bucket->ub_cnt);  in zone_fetch_bucket()
831 		if (zdom->uzd_limin >= bucket->ub_cnt) {  in zone_fetch_bucket()
832 			zdom->uzd_limin -= bucket->ub_cnt;  in zone_fetch_bucket()
834 			zdom->uzd_limin = 0;  in zone_fetch_bucket()
835 			zdom->uzd_timin = 0;  in zone_fetch_bucket()
837 	} else if (zdom->uzd_bimin > zdom->uzd_nitems) {  in zone_fetch_bucket()
838 		zdom->uzd_bimin = zdom->uzd_nitems;  in zone_fetch_bucket()
839 		if (zdom->uzd_imin > zdom->uzd_nitems)  in zone_fetch_bucket()
840 			zdom->uzd_imin = zdom->uzd_nitems;  in zone_fetch_bucket()
843 	ZDOM_UNLOCK(zdom);  in zone_fetch_bucket()
861 	uma_zone_domain_t zdom;  in zone_put_bucket()  local
866 	zdom = zone_domain_lock(zone, domain);  in zone_put_bucket()
871 	zdom->uzd_nitems += bucket->ub_cnt;  in zone_put_bucket()
872 	if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) {  in zone_put_bucket()
874 			zone_domain_imax_set(zdom, zdom->uzd_nitems);  in zone_put_bucket()
880 			atomic_add_long(&zdom->uzd_imax, bucket->ub_cnt);  in zone_put_bucket()
881 			zdom->uzd_imin += bucket->ub_cnt;  in zone_put_bucket()
882 			zdom->uzd_bimin += bucket->ub_cnt;  in zone_put_bucket()
883 			zdom->uzd_limin += bucket->ub_cnt;  in zone_put_bucket()
885 		if (STAILQ_EMPTY(&zdom->uzd_buckets))  in zone_put_bucket()
886 			zdom->uzd_seq = bucket->ub_seq;  in zone_put_bucket()
893 			STAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);  in zone_put_bucket()
895 			STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link);  in zone_put_bucket()
896 		ZDOM_UNLOCK(zdom);  in zone_put_bucket()
899 	zdom->uzd_nitems -= bucket->ub_cnt;  in zone_put_bucket()
900 	ZDOM_UNLOCK(zdom);  in zone_put_bucket()
1050 	uma_zone_domain_t zdom;  in cache_fetch_bucket()  local
1057 	zdom = ZDOM_GET(zone, domain);  in cache_fetch_bucket()
1058 	if (zdom->uzd_nitems == 0)  in cache_fetch_bucket()
1062 	    (seq = atomic_load_32(&zdom->uzd_seq)) != SMR_SEQ_INVALID &&  in cache_fetch_bucket()
1069 	zdom = zone_domain_lock(zone, domain);  in cache_fetch_bucket()
1070 	if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL)  in cache_fetch_bucket()
1072 	ZDOM_UNLOCK(zdom);  in cache_fetch_bucket()
1123 zone_domain_update_wss(uma_zone_domain_t zdom)  in zone_domain_update_wss()  argument
1127 	ZDOM_LOCK_ASSERT(zdom);  in zone_domain_update_wss()
1128 	MPASS(zdom->uzd_imax >= zdom->uzd_nitems);  in zone_domain_update_wss()
1129 	MPASS(zdom->uzd_nitems >= zdom->uzd_bimin);  in zone_domain_update_wss()
1130 	MPASS(zdom->uzd_bimin >= zdom->uzd_imin);  in zone_domain_update_wss()
1136 	zdom->uzd_wss = lmax(zdom->uzd_wss * 3 / 4,  in zone_domain_update_wss()
1137 	    zdom->uzd_imax - zdom->uzd_bimin);  in zone_domain_update_wss()
1146 	m = zdom->uzd_imin - zdom->uzd_wss;  in zone_domain_update_wss()
1148 		if (zdom->uzd_limin >= m)  in zone_domain_update_wss()
1149 			zdom->uzd_limin = m;  in zone_domain_update_wss()
1151 			zdom->uzd_limin = (m + zdom->uzd_limin * 255) / 256;  in zone_domain_update_wss()
1152 		zdom->uzd_timin++;  in zone_domain_update_wss()
1154 		zdom->uzd_limin = 0;  in zone_domain_update_wss()
1155 		zdom->uzd_timin = 0;  in zone_domain_update_wss()
1159 	atomic_subtract_long(&zdom->uzd_imax,  in zone_domain_update_wss()
1160 	    (zdom->uzd_imax - zdom->uzd_nitems + 1) / 2);  in zone_domain_update_wss()
1161 	zdom->uzd_imin = zdom->uzd_bimin = zdom->uzd_nitems;  in zone_domain_update_wss()
1519 	uma_zone_domain_t zdom;  in bucket_cache_reclaim_domain()  local
1528 	zdom = ZDOM_GET(zone, domain);  in bucket_cache_reclaim_domain()
1531 		bucket = zdom->uzd_cross;  in bucket_cache_reclaim_domain()
1532 		zdom->uzd_cross = NULL;  in bucket_cache_reclaim_domain()
1546 	ZDOM_LOCK(zdom);  in bucket_cache_reclaim_domain()
1547 	zone_domain_update_wss(zdom);  in bucket_cache_reclaim_domain()
1551 		target = zdom->uzd_wss;  in bucket_cache_reclaim_domain()
1552 	else if (zdom->uzd_timin > 900 / UMA_TIMEOUT)  in bucket_cache_reclaim_domain()
1553 		target = zdom->uzd_nitems - zdom->uzd_limin / 4;  in bucket_cache_reclaim_domain()
1555 		ZDOM_UNLOCK(zdom);  in bucket_cache_reclaim_domain()
1558 	while ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) != NULL &&  in bucket_cache_reclaim_domain()
1559 	    zdom->uzd_nitems >= target + bucket->ub_cnt) {  in bucket_cache_reclaim_domain()
1560 		bucket = zone_fetch_bucket(zone, zdom, true);  in bucket_cache_reclaim_domain()
1565 		ZDOM_LOCK(zdom);  in bucket_cache_reclaim_domain()
1567 	ZDOM_UNLOCK(zdom);  in bucket_cache_reclaim_domain()
2589 	uma_zone_domain_t zdom;  in zone_alloc_sysctl()  local
2716 		zdom = ZDOM_GET(zone, i);  in zone_alloc_sysctl()
2721 		    "nitems", CTLFLAG_RD, &zdom->uzd_nitems,  in zone_alloc_sysctl()
2724 		    "imax", CTLFLAG_RD, &zdom->uzd_imax,  in zone_alloc_sysctl()
2727 		    "imin", CTLFLAG_RD, &zdom->uzd_imin,  in zone_alloc_sysctl()
2730 		    "bimin", CTLFLAG_RD, &zdom->uzd_bimin,  in zone_alloc_sysctl()
2733 		    "wss", CTLFLAG_RD, &zdom->uzd_wss,  in zone_alloc_sysctl()
2736 		    "limin", CTLFLAG_RD, &zdom->uzd_limin,  in zone_alloc_sysctl()
2739 		    "timin", CTLFLAG_RD, &zdom->uzd_timin, 0,  in zone_alloc_sysctl()
2811 	uma_zone_domain_t zdom;  in zone_ctor()  local
2841 		zdom = ZDOM_GET(zone, i);  in zone_ctor()
2842 		ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS));  in zone_ctor()
2843 		STAILQ_INIT(&zdom->uzd_buckets);  in zone_ctor()
3846 	uma_zone_domain_t zdom;  in uma_zalloc_domain()  local
3878 	zdom = zone_domain_lock(zone, domain);  in uma_zalloc_domain()
3879 	if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) {  in uma_zalloc_domain()
3896 	ZDOM_UNLOCK(zdom);  in uma_zalloc_domain()
4588 	uma_zone_domain_t zdom;  in zone_free_cross()  local
4617 		zdom = ZDOM_GET(zone, domain);  in zone_free_cross()
4618 		if (zdom->uzd_cross == NULL) {  in zone_free_cross()
4621 				zdom->uzd_cross = b;  in zone_free_cross()
4634 				if (zdom->uzd_cross != NULL) {  in zone_free_cross()
4638 					zdom->uzd_cross = b;  in zone_free_cross()
4642 		b = zdom->uzd_cross;  in zone_free_cross()
4649 			zdom->uzd_cross = b;  in zone_free_cross()
5484 	uma_zone_domain_t zdom;  local
5489 		zdom = ZDOM_GET(z, i);
5490 		uth->uth_zone_free += zdom->uzd_nitems;