1 /* $NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $");
34
35 #include <sys/param.h>
36 #include <sys/types.h>
37 #include <sys/bitops.h>
38 #include <sys/condvar.h>
39 #include <sys/mutex.h>
40 #include <sys/systm.h>
41
42 #include <linux/bitops.h>
43 #include <linux/sched.h>
44 #include <linux/wait_bit.h>
45
46 static struct {
47 struct waitbitentry {
48 kmutex_t lock;
49 kcondvar_t cv;
50 } ent;
51 char pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)];
52 } waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned;
53 CTASSERT(sizeof(waitbittab) == PAGE_SIZE);
54 CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE);
55
56 int
linux_wait_bit_init(void)57 linux_wait_bit_init(void)
58 {
59 size_t i;
60
61 for (i = 0; i < __arraycount(waitbittab); i++) {
62 mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM);
63 cv_init(&waitbittab[i].ent.cv, "waitbit");
64 }
65
66 return 0;
67 }
68
69 void
linux_wait_bit_fini(void)70 linux_wait_bit_fini(void)
71 {
72 size_t i;
73
74 for (i = 0; i < __arraycount(waitbittab); i++) {
75 cv_destroy(&waitbittab[i].ent.cv);
76 mutex_destroy(&waitbittab[i].ent.lock);
77 }
78 }
79
80 static inline size_t
wait_bit_hash(const volatile unsigned long * bitmap,unsigned bit)81 wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit)
82 {
83 /* Try to avoid cache line collisions. */
84 const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word));
85
86 return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) %
87 __arraycount(waitbittab);
88 }
89
90 static struct waitbitentry *
wait_bit_enter(const volatile unsigned long * bitmap,unsigned bit)91 wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit)
92 {
93 struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent;
94
95 mutex_enter(&wbe->lock);
96
97 return wbe;
98 }
99
100 static void
wait_bit_exit(struct waitbitentry * wbe)101 wait_bit_exit(struct waitbitentry *wbe)
102 {
103
104 mutex_exit(&wbe->lock);
105 }
106
107 /*
108 * clear_and_wake_up_bit(bit, bitmap)
109 *
110 * Clear the specified bit in the bitmap and wake any waiters in
111 * wait_on_bit or wait_on_bit_timeout that were waiting for it to
112 * clear.
113 */
114 void
clear_and_wake_up_bit(int bit,volatile unsigned long * bitmap)115 clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap)
116 {
117 struct waitbitentry *wbe;
118
119 wbe = wait_bit_enter(bitmap, bit);
120 clear_bit(bit, bitmap);
121 cv_broadcast(&wbe->cv);
122 wait_bit_exit(wbe);
123 }
124
125 /*
126 * wait_on_bit(bitmap, bit, flags)
127 *
128 * Wait for the specified bit in bitmap to be cleared. Returns 0
129 * on success, -EINTR on signal, unless flags has
130 * TASK_UNINTERRUPTIBLE set.
131 */
132 int
wait_on_bit(const volatile unsigned long * bitmap,unsigned bit,int flags)133 wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags)
134 {
135 struct waitbitentry *wbe;
136 int error, ret;
137
138 if (test_bit(bit, bitmap) == 0)
139 return 0;
140
141 wbe = wait_bit_enter(bitmap, bit);
142
143 while (test_bit(bit, bitmap)) {
144 if (flags & TASK_UNINTERRUPTIBLE) {
145 cv_wait(&wbe->cv, &wbe->lock);
146 } else {
147 error = cv_wait_sig(&wbe->cv, &wbe->lock);
148 if (error) {
149 /* cv_wait_sig can only fail on signal. */
150 KASSERTMSG(error == EINTR || error == ERESTART,
151 "error=%d", error);
152 ret = -EINTR;
153 goto out;
154 }
155 }
156 }
157
158 /* Bit is clear. Return zero on success. */
159 KASSERT(test_bit(bit, bitmap) == 0);
160 ret = 0;
161
162 out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
163 wait_bit_exit(wbe);
164 return ret;
165 }
166
167 /*
168 * wait_on_bit_timeout(bitmap, bit, flags, timeout)
169 *
170 * Wait for the specified bit in bitmap to be cleared. Returns 0
171 * on success, -EINTR on signal unless flags has
172 * TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout.
173 */
174 int
wait_on_bit_timeout(const volatile unsigned long * bitmap,unsigned bit,int flags,unsigned long timeout)175 wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit,
176 int flags, unsigned long timeout)
177 {
178 struct waitbitentry *wbe;
179 int error, ret;
180
181 if (test_bit(bit, bitmap) == 0)
182 return 0;
183
184 wbe = wait_bit_enter(bitmap, bit);
185
186 while (test_bit(bit, bitmap)) {
187 unsigned starttime, endtime;
188
189 if (timeout == 0) {
190 ret = -EAGAIN;
191 goto out;
192 }
193
194 starttime = getticks();
195 if (flags & TASK_UNINTERRUPTIBLE) {
196 error = cv_timedwait(&wbe->cv, &wbe->lock,
197 MIN(timeout, INT_MAX/2));
198 } else {
199 error = cv_timedwait_sig(&wbe->cv, &wbe->lock,
200 MIN(timeout, INT_MAX/2));
201 }
202 endtime = getticks();
203
204 /*
205 * If we were interrupted or timed out, massage the
206 * error return and stop here.
207 */
208 if (error) {
209 KASSERTMSG((error == EINTR || error == ERESTART ||
210 error == EWOULDBLOCK), "error=%d", error);
211 if (error == EINTR || error == ERESTART) {
212 ret = -EINTR;
213 } else if (error == EWOULDBLOCK) {
214 ret = -EAGAIN;
215 } else {
216 panic("invalid error=%d", error);
217 }
218 goto out;
219 }
220
221 /* Otherwise, debit the time spent. */
222 timeout -= MIN(timeout, (endtime - starttime));
223 }
224
225 /* Bit is clear. Return zero on success. */
226 KASSERT(test_bit(bit, bitmap) == 0);
227 ret = timeout;
228
229 out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
230 wait_bit_exit(wbe);
231 return ret;
232 }
233