1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2001 Matthew Dillon. All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 /* Mutex pool routines. These routines are designed to be used as short
29 * term leaf mutexes (e.g. the last mutex you might acquire other then
30 * calling msleep()). They operate using a shared pool. A mutex is chosen
31 * from the pool based on the supplied pointer (which may or may not be
32 * valid).
33 *
34 * Advantages:
35 * - no structural overhead. Mutexes can be associated with structures
36 * without adding bloat to the structures.
37 * - mutexes can be obtained for invalid pointers, useful when uses
38 * mutexes to interlock destructor ops.
39 * - no initialization/destructor overhead.
40 * - can be used with msleep.
41 *
42 * Disadvantages:
43 * - should generally only be used as leaf mutexes.
44 * - pool/pool dependency ordering cannot be depended on.
45 * - possible L1 cache mastersip contention between cpus.
46 */
47
48 #include <sys/cdefs.h>
49 #include <sys/param.h>
50 #include <sys/proc.h>
51 #include <sys/kernel.h>
52 #include <sys/ktr.h>
53 #include <sys/lock.h>
54 #include <sys/malloc.h>
55 #include <sys/mutex.h>
56 #include <sys/systm.h>
57
58 static MALLOC_DEFINE(M_MTXPOOL, "mtx_pool", "mutex pool");
59
60 /* Pool sizes must be a power of two */
61 #ifndef MTX_POOL_SLEEP_SIZE
62 #define MTX_POOL_SLEEP_SIZE 1024
63 #endif
64
65 struct mtxpool_header {
66 int mtxpool_size;
67 int mtxpool_mask;
68 int mtxpool_shift;
69 int mtxpool_next __aligned(CACHE_LINE_SIZE);
70 };
71
72 struct mtx_pool {
73 struct mtxpool_header mtx_pool_header;
74 struct mtx mtx_pool_ary[1];
75 };
76
77 #define mtx_pool_size mtx_pool_header.mtxpool_size
78 #define mtx_pool_mask mtx_pool_header.mtxpool_mask
79 #define mtx_pool_shift mtx_pool_header.mtxpool_shift
80 #define mtx_pool_next mtx_pool_header.mtxpool_next
81
82 struct mtx_pool __read_mostly *mtxpool_sleep;
83
84 #if UINTPTR_MAX == UINT64_MAX /* 64 bits */
85 # define POINTER_BITS 64
86 # define HASH_MULTIPLIER 11400714819323198485u /* (2^64)*(sqrt(5)-1)/2 */
87 #else /* assume 32 bits */
88 # define POINTER_BITS 32
89 # define HASH_MULTIPLIER 2654435769u /* (2^32)*(sqrt(5)-1)/2 */
90 #endif
91
92 /*
93 * Return the (shared) pool mutex associated with the specified address.
94 * The returned mutex is a leaf level mutex, meaning that if you obtain it
95 * you cannot obtain any other mutexes until you release it. You can
96 * legally msleep() on the mutex.
97 */
98 struct mtx *
mtx_pool_find(struct mtx_pool * pool,void * ptr)99 mtx_pool_find(struct mtx_pool *pool, void *ptr)
100 {
101 int p;
102
103 KASSERT(pool != NULL, ("_mtx_pool_find(): null pool"));
104 /*
105 * Fibonacci hash, see Knuth's
106 * _Art of Computer Programming, Volume 3 / Sorting and Searching_
107 */
108 p = ((HASH_MULTIPLIER * (uintptr_t)ptr) >> pool->mtx_pool_shift) &
109 pool->mtx_pool_mask;
110 return (&pool->mtx_pool_ary[p]);
111 }
112
113 static void
mtx_pool_initialize(struct mtx_pool * pool,const char * mtx_name,int pool_size,int opts)114 mtx_pool_initialize(struct mtx_pool *pool, const char *mtx_name, int pool_size,
115 int opts)
116 {
117 int i, maskbits;
118
119 pool->mtx_pool_size = pool_size;
120 pool->mtx_pool_mask = pool_size - 1;
121 for (i = 1, maskbits = 0; (i & pool_size) == 0; i = i << 1)
122 maskbits++;
123 pool->mtx_pool_shift = POINTER_BITS - maskbits;
124 pool->mtx_pool_next = 0;
125 for (i = 0; i < pool_size; ++i)
126 mtx_init(&pool->mtx_pool_ary[i], mtx_name, NULL, opts);
127 }
128
129 struct mtx_pool *
mtx_pool_create(const char * mtx_name,int pool_size,int opts)130 mtx_pool_create(const char *mtx_name, int pool_size, int opts)
131 {
132 struct mtx_pool *pool;
133
134 if (pool_size <= 0 || !powerof2(pool_size)) {
135 printf("WARNING: %s pool size is not a power of 2.\n",
136 mtx_name);
137 pool_size = 128;
138 }
139 pool = malloc(sizeof (struct mtx_pool) +
140 ((pool_size - 1) * sizeof (struct mtx)),
141 M_MTXPOOL, M_WAITOK | M_ZERO);
142 mtx_pool_initialize(pool, mtx_name, pool_size, opts);
143 return pool;
144 }
145
146 void
mtx_pool_destroy(struct mtx_pool ** poolp)147 mtx_pool_destroy(struct mtx_pool **poolp)
148 {
149 int i;
150 struct mtx_pool *pool = *poolp;
151
152 for (i = pool->mtx_pool_size - 1; i >= 0; --i)
153 mtx_destroy(&pool->mtx_pool_ary[i]);
154 free(pool, M_MTXPOOL);
155 *poolp = NULL;
156 }
157
158 static void
mtx_pool_setup_dynamic(void * dummy __unused)159 mtx_pool_setup_dynamic(void *dummy __unused)
160 {
161 mtxpool_sleep = mtx_pool_create("sleep mtxpool",
162 MTX_POOL_SLEEP_SIZE, MTX_DEF);
163 }
164
165 /*
166 * Obtain a (shared) mutex from the pool. The returned mutex is a leaf
167 * level mutex, meaning that if you obtain it you cannot obtain any other
168 * mutexes until you release it. You can legally msleep() on the mutex.
169 */
170 struct mtx *
mtx_pool_alloc(struct mtx_pool * pool)171 mtx_pool_alloc(struct mtx_pool *pool)
172 {
173 int i;
174
175 KASSERT(pool != NULL, ("mtx_pool_alloc(): null pool"));
176 /*
177 * mtx_pool_next is unprotected against multiple accesses,
178 * but simultaneous access by two CPUs should not be very
179 * harmful.
180 */
181 i = pool->mtx_pool_next;
182 pool->mtx_pool_next = (i + 1) & pool->mtx_pool_mask;
183 return (&pool->mtx_pool_ary[i]);
184 }
185
186 SYSINIT(mtxpooli2, SI_SUB_MTX_POOL_DYNAMIC, SI_ORDER_FIRST,
187 mtx_pool_setup_dynamic, NULL);
188