1 /* $NetBSD: linux_rwsem.c,v 1.4 2021/12/19 11:21:54 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2021 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: linux_rwsem.c,v 1.4 2021/12/19 11:21:54 riastradh Exp $");
31
32 #include <sys/types.h>
33
34 #include <sys/condvar.h>
35 #include <sys/lwp.h>
36 #include <sys/lockdebug.h>
37 #include <sys/rwlock.h>
38
39 #include <machine/limits.h>
40
41 #include <lib/libkern/libkern.h>
42
43 #include <linux/rwsem.h>
44
45 #define RWSEM_WANTLOCK(RWSEM) \
46 LOCKDEBUG_WANTLOCK((RWSEM)->rws_debug, (RWSEM), \
47 (uintptr_t)__builtin_return_address(0), 0)
48 #define RWSEM_LOCKED_EX(RWSEM) \
49 LOCKDEBUG_LOCKED((RWSEM)->rws_debug, (RWSEM), NULL, \
50 (uintptr_t)__builtin_return_address(0), 0)
51 #define RWSEM_LOCKED_SH(RWSEM) \
52 LOCKDEBUG_LOCKED((RWSEM)->rws_debug, (RWSEM), NULL, \
53 (uintptr_t)__builtin_return_address(0), 1)
54 #define RWSEM_UNLOCKED_EX(RWSEM) \
55 LOCKDEBUG_UNLOCKED((RWSEM)->rws_debug, (RWSEM), \
56 (uintptr_t)__builtin_return_address(0), 0)
57 #define RWSEM_UNLOCKED_SH(RWSEM) \
58 LOCKDEBUG_UNLOCKED((RWSEM)->rws_debug, (RWSEM), \
59 (uintptr_t)__builtin_return_address(0), 1)
60
61 #ifdef LOCKDEBUG
62 static void
rwsem_dump(const volatile void * cookie,lockop_printer_t pr)63 rwsem_dump(const volatile void *cookie, lockop_printer_t pr)
64 {
65 const volatile struct rw_semaphore *rwsem = cookie;
66
67 pr("%-13s: %p", "writer", rwsem->rws_writer);
68 pr("%-13s: %u", "readers", rwsem->rws_readers);
69 pr("%-13s: %s", "writewanted", rwsem->rws_writewanted ? "yes" : "no");
70 }
71
72 static lockops_t rwsem_lockops = {
73 .lo_name = "Linux read/write semaphore",
74 .lo_type = LOCKOPS_SLEEP,
75 .lo_dump = rwsem_dump,
76 };
77 #endif
78
79 void
init_rwsem(struct rw_semaphore * rwsem)80 init_rwsem(struct rw_semaphore *rwsem)
81 {
82
83 mutex_init(&rwsem->rws_lock, MUTEX_DEFAULT, IPL_VM);
84 cv_init(&rwsem->rws_cv, "lnxrwsem");
85 rwsem->rws_writer = NULL;
86 rwsem->rws_readers = 0;
87
88 #ifdef LOCKDEBUG
89 rwsem->rws_debug = LOCKDEBUG_ALLOC(rwsem, &rwsem_lockops,
90 (uintptr_t)__builtin_return_address(0));
91 #endif
92 }
93
94 void
destroy_rwsem(struct rw_semaphore * rwsem)95 destroy_rwsem(struct rw_semaphore *rwsem)
96 {
97
98 KASSERT(rwsem->rws_readers == 0);
99 KASSERT(rwsem->rws_writer == NULL);
100
101 #ifdef LOCKDEBUG
102 LOCKDEBUG_FREE(rwsem->rws_debug, rwsem);
103 #endif
104
105 cv_destroy(&rwsem->rws_cv);
106 mutex_destroy(&rwsem->rws_lock);
107 }
108
109 void
down_read(struct rw_semaphore * rwsem)110 down_read(struct rw_semaphore *rwsem)
111 {
112
113 RWSEM_WANTLOCK(rwsem);
114
115 mutex_enter(&rwsem->rws_lock);
116 while (rwsem->rws_writer || rwsem->rws_writewanted)
117 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock);
118 KASSERT(rwsem->rws_readers < UINT_MAX);
119 rwsem->rws_readers++;
120 mutex_exit(&rwsem->rws_lock);
121
122 RWSEM_LOCKED_SH(rwsem);
123 }
124
125 bool
down_read_trylock(struct rw_semaphore * rwsem)126 down_read_trylock(struct rw_semaphore *rwsem)
127 {
128 bool ret = false;
129
130 /*
131 * Note: Linux apparently relies on down_read_trylock to
132 * quietly succeed when the caller already holds a reader lock.
133 * This is why we can't use rwlock(9), which absolutely
134 * prohibits recursive use and crashes immediately under
135 * LOCKDEBUG if you try it.
136 */
137
138 mutex_enter(&rwsem->rws_lock);
139 if (rwsem->rws_writer == NULL && !rwsem->rws_writewanted) {
140 KASSERT(rwsem->rws_readers < UINT_MAX);
141 rwsem->rws_readers++;
142 ret = true;
143 }
144 mutex_exit(&rwsem->rws_lock);
145
146 if (ret) {
147 RWSEM_LOCKED_SH(rwsem);
148 }
149
150 return ret;
151 }
152
153 void
up_read(struct rw_semaphore * rwsem)154 up_read(struct rw_semaphore *rwsem)
155 {
156
157 RWSEM_UNLOCKED_SH(rwsem);
158
159 mutex_enter(&rwsem->rws_lock);
160 KASSERT(rwsem->rws_readers);
161 KASSERT(rwsem->rws_writer == NULL);
162 if (--rwsem->rws_readers == 0)
163 cv_broadcast(&rwsem->rws_cv);
164 mutex_exit(&rwsem->rws_lock);
165 }
166
167 void
down_write(struct rw_semaphore * rwsem)168 down_write(struct rw_semaphore *rwsem)
169 {
170
171 RWSEM_WANTLOCK(rwsem);
172
173 mutex_enter(&rwsem->rws_lock);
174
175 /* If another writer is waiting, get in the queue. */
176 while (rwsem->rws_writewanted)
177 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock);
178
179 /*
180 * No other writers waiting. Our turn. Announce our intent to
181 * readers, and wait for the writer or readers to finish.
182 */
183 rwsem->rws_writewanted = true;
184 while (rwsem->rws_writer || rwsem->rws_readers) {
185 KASSERTMSG(rwsem->rws_writer != curlwp,
186 "locking against myself: rwsem=%p lwp=%p", rwsem, curlwp);
187 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock);
188 }
189
190 /* At last, it is ours! */
191 KASSERT(rwsem->rws_readers == 0);
192 KASSERT(rwsem->rws_writer == NULL);
193 KASSERT(rwsem->rws_writewanted);
194 rwsem->rws_writewanted = false;
195 rwsem->rws_writer = curlwp;
196
197 mutex_exit(&rwsem->rws_lock);
198
199 RWSEM_LOCKED_EX(rwsem);
200 }
201
202 void
up_write(struct rw_semaphore * rwsem)203 up_write(struct rw_semaphore *rwsem)
204 {
205
206 RWSEM_UNLOCKED_EX(rwsem);
207
208 mutex_enter(&rwsem->rws_lock);
209 KASSERT(rwsem->rws_writer == curlwp);
210 KASSERT(rwsem->rws_readers == 0);
211 rwsem->rws_writer = NULL;
212 cv_broadcast(&rwsem->rws_cv);
213 mutex_exit(&rwsem->rws_lock);
214 }
215
216 void
downgrade_write(struct rw_semaphore * rwsem)217 downgrade_write(struct rw_semaphore *rwsem)
218 {
219
220 RWSEM_UNLOCKED_EX(rwsem);
221
222 mutex_enter(&rwsem->rws_lock);
223 KASSERT(rwsem->rws_writer == curlwp);
224 KASSERT(rwsem->rws_readers == 0);
225 rwsem->rws_writer = NULL;
226 rwsem->rws_readers = 1;
227 cv_broadcast(&rwsem->rws_cv);
228 mutex_exit(&rwsem->rws_lock);
229
230 RWSEM_LOCKED_SH(rwsem);
231 }
232