1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015, 2016 The FreeBSD Foundation
5 * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
6 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
7 * All rights reserved.
8 *
9 * Portions of this software were developed by Konstantin Belousov
10 * under sponsorship from the FreeBSD Foundation.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice unmodified, this list of conditions, and the following
17 * disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_umtx_profiling.h"
36
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/fcntl.h>
40 #include <sys/file.h>
41 #include <sys/filedesc.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mman.h>
46 #include <sys/mutex.h>
47 #include <sys/priv.h>
48 #include <sys/proc.h>
49 #include <sys/resource.h>
50 #include <sys/resourcevar.h>
51 #include <sys/rwlock.h>
52 #include <sys/sbuf.h>
53 #include <sys/sched.h>
54 #include <sys/smp.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/sysproto.h>
58 #include <sys/syscallsubr.h>
59 #include <sys/taskqueue.h>
60 #include <sys/time.h>
61 #include <sys/eventhandler.h>
62 #include <sys/umtx.h>
63 #include <sys/umtxvar.h>
64
65 #include <security/mac/mac_framework.h>
66
67 #include <vm/vm.h>
68 #include <vm/vm_param.h>
69 #include <vm/pmap.h>
70 #include <vm/vm_map.h>
71 #include <vm/vm_object.h>
72
73 #include <machine/atomic.h>
74 #include <machine/cpu.h>
75
76 #include <compat/freebsd32/freebsd32.h>
77 #ifdef COMPAT_FREEBSD32
78 #include <compat/freebsd32/freebsd32_proto.h>
79 #endif
80
81 #define _UMUTEX_TRY 1
82 #define _UMUTEX_WAIT 2
83
84 #ifdef UMTX_PROFILING
85 #define UPROF_PERC_BIGGER(w, f, sw, sf) \
86 (((w) > (sw)) || ((w) == (sw) && (f) > (sf)))
87 #endif
88
89 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
90 #ifdef INVARIANTS
91 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do { \
92 struct umtxq_chain *uc; \
93 \
94 uc = umtxq_getchain(key); \
95 mtx_assert(&uc->uc_lock, MA_OWNED); \
96 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \
97 } while (0)
98 #else
99 #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0)
100 #endif
101
102 /*
103 * Don't propagate time-sharing priority, there is a security reason,
104 * a user can simply introduce PI-mutex, let thread A lock the mutex,
105 * and let another thread B block on the mutex, because B is
106 * sleeping, its priority will be boosted, this causes A's priority to
107 * be boosted via priority propagating too and will never be lowered even
108 * if it is using 100%CPU, this is unfair to other processes.
109 */
110
111 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
112 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
113 PRI_MAX_TIMESHARE : (td)->td_user_pri)
114
115 #define GOLDEN_RATIO_PRIME 2654404609U
116 #ifndef UMTX_CHAINS
117 #define UMTX_CHAINS 512
118 #endif
119 #define UMTX_SHIFTS (__WORD_BIT - 9)
120
121 #define GET_SHARE(flags) \
122 (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
123
124 #define BUSY_SPINS 200
125
126 struct umtx_copyops {
127 int (*copyin_timeout)(const void *uaddr, struct timespec *tsp);
128 int (*copyin_umtx_time)(const void *uaddr, size_t size,
129 struct _umtx_time *tp);
130 int (*copyin_robust_lists)(const void *uaddr, size_t size,
131 struct umtx_robust_lists_params *rbp);
132 int (*copyout_timeout)(void *uaddr, size_t size,
133 struct timespec *tsp);
134 const size_t timespec_sz;
135 const size_t umtx_time_sz;
136 const bool compat32;
137 };
138
139 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32");
140 _Static_assert(__offsetof(struct umutex, m_spare[0]) ==
141 __offsetof(struct umutex32, m_spare[0]), "m_spare32");
142
143 int umtx_shm_vnobj_persistent = 0;
144 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN,
145 &umtx_shm_vnobj_persistent, 0,
146 "False forces destruction of umtx attached to file, on last close");
147 static int umtx_max_rb = 1000;
148 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN,
149 &umtx_max_rb, 0,
150 "Maximum number of robust mutexes allowed for each thread");
151
152 static uma_zone_t umtx_pi_zone;
153 static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS];
154 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
155 static int umtx_pi_allocated;
156
157 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
158 "umtx debug");
159 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
160 &umtx_pi_allocated, 0, "Allocated umtx_pi");
161 static int umtx_verbose_rb = 1;
162 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN,
163 &umtx_verbose_rb, 0,
164 "");
165
166 #ifdef UMTX_PROFILING
167 static long max_length;
168 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
169 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
170 "umtx chain stats");
171 #endif
172
173 static inline void umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
174 const struct _umtx_time *umtxtime);
175
176 static void umtx_shm_init(void);
177 static void umtxq_sysinit(void *);
178 static void umtxq_hash(struct umtx_key *key);
179 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags,
180 bool rb);
181 static void umtx_thread_cleanup(struct thread *td);
182 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
183
184 #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
185
186 static struct mtx umtx_lock;
187
188 #ifdef UMTX_PROFILING
189 static void
umtx_init_profiling(void)190 umtx_init_profiling(void)
191 {
192 struct sysctl_oid *chain_oid;
193 char chain_name[10];
194 int i;
195
196 for (i = 0; i < UMTX_CHAINS; ++i) {
197 snprintf(chain_name, sizeof(chain_name), "%d", i);
198 chain_oid = SYSCTL_ADD_NODE(NULL,
199 SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
200 chain_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
201 "umtx hash stats");
202 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
203 "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
204 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
205 "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
206 }
207 }
208
209 static int
sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)210 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)
211 {
212 char buf[512];
213 struct sbuf sb;
214 struct umtxq_chain *uc;
215 u_int fract, i, j, tot, whole;
216 u_int sf0, sf1, sf2, sf3, sf4;
217 u_int si0, si1, si2, si3, si4;
218 u_int sw0, sw1, sw2, sw3, sw4;
219
220 sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN);
221 for (i = 0; i < 2; i++) {
222 tot = 0;
223 for (j = 0; j < UMTX_CHAINS; ++j) {
224 uc = &umtxq_chains[i][j];
225 mtx_lock(&uc->uc_lock);
226 tot += uc->max_length;
227 mtx_unlock(&uc->uc_lock);
228 }
229 if (tot == 0)
230 sbuf_printf(&sb, "%u) Empty ", i);
231 else {
232 sf0 = sf1 = sf2 = sf3 = sf4 = 0;
233 si0 = si1 = si2 = si3 = si4 = 0;
234 sw0 = sw1 = sw2 = sw3 = sw4 = 0;
235 for (j = 0; j < UMTX_CHAINS; j++) {
236 uc = &umtxq_chains[i][j];
237 mtx_lock(&uc->uc_lock);
238 whole = uc->max_length * 100;
239 mtx_unlock(&uc->uc_lock);
240 fract = (whole % tot) * 100;
241 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) {
242 sf0 = fract;
243 si0 = j;
244 sw0 = whole;
245 } else if (UPROF_PERC_BIGGER(whole, fract, sw1,
246 sf1)) {
247 sf1 = fract;
248 si1 = j;
249 sw1 = whole;
250 } else if (UPROF_PERC_BIGGER(whole, fract, sw2,
251 sf2)) {
252 sf2 = fract;
253 si2 = j;
254 sw2 = whole;
255 } else if (UPROF_PERC_BIGGER(whole, fract, sw3,
256 sf3)) {
257 sf3 = fract;
258 si3 = j;
259 sw3 = whole;
260 } else if (UPROF_PERC_BIGGER(whole, fract, sw4,
261 sf4)) {
262 sf4 = fract;
263 si4 = j;
264 sw4 = whole;
265 }
266 }
267 sbuf_printf(&sb, "queue %u:\n", i);
268 sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot,
269 sf0 / tot, si0);
270 sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot,
271 sf1 / tot, si1);
272 sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot,
273 sf2 / tot, si2);
274 sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot,
275 sf3 / tot, si3);
276 sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot,
277 sf4 / tot, si4);
278 }
279 }
280 sbuf_trim(&sb);
281 sbuf_finish(&sb);
282 sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
283 sbuf_delete(&sb);
284 return (0);
285 }
286
287 static int
sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)288 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)
289 {
290 struct umtxq_chain *uc;
291 u_int i, j;
292 int clear, error;
293
294 clear = 0;
295 error = sysctl_handle_int(oidp, &clear, 0, req);
296 if (error != 0 || req->newptr == NULL)
297 return (error);
298
299 if (clear != 0) {
300 for (i = 0; i < 2; ++i) {
301 for (j = 0; j < UMTX_CHAINS; ++j) {
302 uc = &umtxq_chains[i][j];
303 mtx_lock(&uc->uc_lock);
304 uc->length = 0;
305 uc->max_length = 0;
306 mtx_unlock(&uc->uc_lock);
307 }
308 }
309 }
310 return (0);
311 }
312
313 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear,
314 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
315 sysctl_debug_umtx_chains_clear, "I",
316 "Clear umtx chains statistics");
317 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks,
318 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
319 sysctl_debug_umtx_chains_peaks, "A",
320 "Highest peaks in chains max length");
321 #endif
322
323 static void
umtxq_sysinit(void * arg __unused)324 umtxq_sysinit(void *arg __unused)
325 {
326 int i, j;
327
328 umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
329 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
330 for (i = 0; i < 2; ++i) {
331 for (j = 0; j < UMTX_CHAINS; ++j) {
332 mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
333 MTX_DEF | MTX_DUPOK);
334 LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
335 LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
336 LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
337 TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
338 umtxq_chains[i][j].uc_busy = 0;
339 umtxq_chains[i][j].uc_waiters = 0;
340 #ifdef UMTX_PROFILING
341 umtxq_chains[i][j].length = 0;
342 umtxq_chains[i][j].max_length = 0;
343 #endif
344 }
345 }
346 #ifdef UMTX_PROFILING
347 umtx_init_profiling();
348 #endif
349 mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF);
350 umtx_shm_init();
351 }
352
353 struct umtx_q *
umtxq_alloc(void)354 umtxq_alloc(void)
355 {
356 struct umtx_q *uq;
357
358 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
359 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
360 M_WAITOK | M_ZERO);
361 TAILQ_INIT(&uq->uq_spare_queue->head);
362 TAILQ_INIT(&uq->uq_pi_contested);
363 uq->uq_inherited_pri = PRI_MAX;
364 return (uq);
365 }
366
367 void
umtxq_free(struct umtx_q * uq)368 umtxq_free(struct umtx_q *uq)
369 {
370
371 MPASS(uq->uq_spare_queue != NULL);
372 free(uq->uq_spare_queue, M_UMTX);
373 free(uq, M_UMTX);
374 }
375
376 static inline void
umtxq_hash(struct umtx_key * key)377 umtxq_hash(struct umtx_key *key)
378 {
379 unsigned n;
380
381 n = (uintptr_t)key->info.both.a + key->info.both.b;
382 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
383 }
384
385 struct umtxq_chain *
umtxq_getchain(struct umtx_key * key)386 umtxq_getchain(struct umtx_key *key)
387 {
388
389 if (key->type <= TYPE_SEM)
390 return (&umtxq_chains[1][key->hash]);
391 return (&umtxq_chains[0][key->hash]);
392 }
393
394 /*
395 * Set chain to busy state when following operation
396 * may be blocked (kernel mutex can not be used).
397 */
398 void
umtxq_busy(struct umtx_key * key)399 umtxq_busy(struct umtx_key *key)
400 {
401 struct umtxq_chain *uc;
402
403 uc = umtxq_getchain(key);
404 mtx_assert(&uc->uc_lock, MA_OWNED);
405 if (uc->uc_busy) {
406 #ifdef SMP
407 if (smp_cpus > 1) {
408 int count = BUSY_SPINS;
409 if (count > 0) {
410 umtxq_unlock(key);
411 while (uc->uc_busy && --count > 0)
412 cpu_spinwait();
413 umtxq_lock(key);
414 }
415 }
416 #endif
417 while (uc->uc_busy) {
418 uc->uc_waiters++;
419 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
420 uc->uc_waiters--;
421 }
422 }
423 uc->uc_busy = 1;
424 }
425
426 /*
427 * Unbusy a chain.
428 */
429 void
umtxq_unbusy(struct umtx_key * key)430 umtxq_unbusy(struct umtx_key *key)
431 {
432 struct umtxq_chain *uc;
433
434 uc = umtxq_getchain(key);
435 mtx_assert(&uc->uc_lock, MA_OWNED);
436 KASSERT(uc->uc_busy != 0, ("not busy"));
437 uc->uc_busy = 0;
438 if (uc->uc_waiters)
439 wakeup_one(uc);
440 }
441
442 void
umtxq_busy_unlocked(struct umtx_key * key)443 umtxq_busy_unlocked(struct umtx_key *key)
444 {
445 umtxq_lock(key);
446 umtxq_busy(key);
447 umtxq_unlock(key);
448 }
449
450 void
umtxq_unbusy_unlocked(struct umtx_key * key)451 umtxq_unbusy_unlocked(struct umtx_key *key)
452 {
453 umtxq_lock(key);
454 umtxq_unbusy(key);
455 umtxq_unlock(key);
456 }
457
458 static struct umtxq_queue *
umtxq_queue_lookup(struct umtx_key * key,int q)459 umtxq_queue_lookup(struct umtx_key *key, int q)
460 {
461 struct umtxq_queue *uh;
462 struct umtxq_chain *uc;
463
464 uc = umtxq_getchain(key);
465 UMTXQ_LOCKED_ASSERT(uc);
466 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
467 if (umtx_key_match(&uh->key, key))
468 return (uh);
469 }
470
471 return (NULL);
472 }
473
474 void
umtxq_insert_queue(struct umtx_q * uq,int q)475 umtxq_insert_queue(struct umtx_q *uq, int q)
476 {
477 struct umtxq_queue *uh;
478 struct umtxq_chain *uc;
479
480 uc = umtxq_getchain(&uq->uq_key);
481 UMTXQ_LOCKED_ASSERT(uc);
482 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
483 uh = umtxq_queue_lookup(&uq->uq_key, q);
484 if (uh != NULL) {
485 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
486 } else {
487 uh = uq->uq_spare_queue;
488 uh->key = uq->uq_key;
489 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
490 #ifdef UMTX_PROFILING
491 uc->length++;
492 if (uc->length > uc->max_length) {
493 uc->max_length = uc->length;
494 if (uc->max_length > max_length)
495 max_length = uc->max_length;
496 }
497 #endif
498 }
499 uq->uq_spare_queue = NULL;
500
501 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
502 uh->length++;
503 uq->uq_flags |= UQF_UMTXQ;
504 uq->uq_cur_queue = uh;
505 return;
506 }
507
508 void
umtxq_remove_queue(struct umtx_q * uq,int q)509 umtxq_remove_queue(struct umtx_q *uq, int q)
510 {
511 struct umtxq_chain *uc;
512 struct umtxq_queue *uh;
513
514 uc = umtxq_getchain(&uq->uq_key);
515 UMTXQ_LOCKED_ASSERT(uc);
516 if (uq->uq_flags & UQF_UMTXQ) {
517 uh = uq->uq_cur_queue;
518 TAILQ_REMOVE(&uh->head, uq, uq_link);
519 uh->length--;
520 uq->uq_flags &= ~UQF_UMTXQ;
521 if (TAILQ_EMPTY(&uh->head)) {
522 KASSERT(uh->length == 0,
523 ("inconsistent umtxq_queue length"));
524 #ifdef UMTX_PROFILING
525 uc->length--;
526 #endif
527 LIST_REMOVE(uh, link);
528 } else {
529 uh = LIST_FIRST(&uc->uc_spare_queue);
530 KASSERT(uh != NULL, ("uc_spare_queue is empty"));
531 LIST_REMOVE(uh, link);
532 }
533 uq->uq_spare_queue = uh;
534 uq->uq_cur_queue = NULL;
535 }
536 }
537
538 /*
539 * Check if there are multiple waiters
540 */
541 int
umtxq_count(struct umtx_key * key)542 umtxq_count(struct umtx_key *key)
543 {
544 struct umtxq_queue *uh;
545
546 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
547 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
548 if (uh != NULL)
549 return (uh->length);
550 return (0);
551 }
552
553 /*
554 * Check if there are multiple PI waiters and returns first
555 * waiter.
556 */
557 static int
umtxq_count_pi(struct umtx_key * key,struct umtx_q ** first)558 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
559 {
560 struct umtxq_queue *uh;
561
562 *first = NULL;
563 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
564 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
565 if (uh != NULL) {
566 *first = TAILQ_FIRST(&uh->head);
567 return (uh->length);
568 }
569 return (0);
570 }
571
572 /*
573 * Wake up threads waiting on an userland object by a bit mask.
574 */
575 int
umtxq_signal_mask(struct umtx_key * key,int n_wake,u_int bitset)576 umtxq_signal_mask(struct umtx_key *key, int n_wake, u_int bitset)
577 {
578 struct umtxq_queue *uh;
579 struct umtx_q *uq, *uq_temp;
580 int ret;
581
582 ret = 0;
583 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
584 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
585 if (uh == NULL)
586 return (0);
587 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
588 if ((uq->uq_bitset & bitset) == 0)
589 continue;
590 umtxq_remove_queue(uq, UMTX_SHARED_QUEUE);
591 wakeup_one(uq);
592 if (++ret >= n_wake)
593 break;
594 }
595 return (ret);
596 }
597
598 /*
599 * Wake up threads waiting on an userland object.
600 */
601
602 static int
umtxq_signal_queue(struct umtx_key * key,int n_wake,int q)603 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
604 {
605 struct umtxq_queue *uh;
606 struct umtx_q *uq;
607 int ret;
608
609 ret = 0;
610 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
611 uh = umtxq_queue_lookup(key, q);
612 if (uh != NULL) {
613 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
614 umtxq_remove_queue(uq, q);
615 wakeup(uq);
616 if (++ret >= n_wake)
617 return (ret);
618 }
619 }
620 return (ret);
621 }
622
623 /*
624 * Wake up specified thread.
625 */
626 static inline void
umtxq_signal_thread(struct umtx_q * uq)627 umtxq_signal_thread(struct umtx_q *uq)
628 {
629
630 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
631 umtxq_remove(uq);
632 wakeup(uq);
633 }
634
635 /*
636 * Wake up a maximum of n_wake threads that are waiting on an userland
637 * object identified by key. The remaining threads are removed from queue
638 * identified by key and added to the queue identified by key2 (requeued).
639 * The n_requeue specifies an upper limit on the number of threads that
640 * are requeued to the second queue.
641 */
642 int
umtxq_requeue(struct umtx_key * key,int n_wake,struct umtx_key * key2,int n_requeue)643 umtxq_requeue(struct umtx_key *key, int n_wake, struct umtx_key *key2,
644 int n_requeue)
645 {
646 struct umtxq_queue *uh;
647 struct umtx_q *uq, *uq_temp;
648 int ret;
649
650 ret = 0;
651 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
652 UMTXQ_LOCKED_ASSERT(umtxq_getchain(key2));
653 uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
654 if (uh == NULL)
655 return (0);
656 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
657 if (++ret <= n_wake) {
658 umtxq_remove(uq);
659 wakeup_one(uq);
660 } else {
661 umtxq_remove(uq);
662 uq->uq_key = *key2;
663 umtxq_insert(uq);
664 if (ret - n_wake == n_requeue)
665 break;
666 }
667 }
668 return (ret);
669 }
670
671 static inline int
tstohz(const struct timespec * tsp)672 tstohz(const struct timespec *tsp)
673 {
674 struct timeval tv;
675
676 TIMESPEC_TO_TIMEVAL(&tv, tsp);
677 return tvtohz(&tv);
678 }
679
680 void
umtx_abs_timeout_init(struct umtx_abs_timeout * timo,int clockid,int absolute,const struct timespec * timeout)681 umtx_abs_timeout_init(struct umtx_abs_timeout *timo, int clockid,
682 int absolute, const struct timespec *timeout)
683 {
684
685 timo->clockid = clockid;
686 if (!absolute) {
687 timo->is_abs_real = false;
688 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
689 timespecadd(&timo->cur, timeout, &timo->end);
690 } else {
691 timo->end = *timeout;
692 timo->is_abs_real = clockid == CLOCK_REALTIME ||
693 clockid == CLOCK_REALTIME_FAST ||
694 clockid == CLOCK_REALTIME_PRECISE ||
695 clockid == CLOCK_SECOND;
696 }
697 }
698
699 static void
umtx_abs_timeout_init2(struct umtx_abs_timeout * timo,const struct _umtx_time * umtxtime)700 umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
701 const struct _umtx_time *umtxtime)
702 {
703
704 umtx_abs_timeout_init(timo, umtxtime->_clockid,
705 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
706 }
707
708 static void
umtx_abs_timeout_enforce_min(sbintime_t * sbt)709 umtx_abs_timeout_enforce_min(sbintime_t *sbt)
710 {
711 sbintime_t when, mint;
712
713 mint = curproc->p_umtx_min_timeout;
714 if (__predict_false(mint != 0)) {
715 when = sbinuptime() + mint;
716 if (*sbt < when)
717 *sbt = when;
718 }
719 }
720
721 static int
umtx_abs_timeout_getsbt(struct umtx_abs_timeout * timo,sbintime_t * sbt,int * flags)722 umtx_abs_timeout_getsbt(struct umtx_abs_timeout *timo, sbintime_t *sbt,
723 int *flags)
724 {
725 struct bintime bt, bbt;
726 struct timespec tts;
727 sbintime_t rem;
728
729 switch (timo->clockid) {
730
731 /* Clocks that can be converted into absolute time. */
732 case CLOCK_REALTIME:
733 case CLOCK_REALTIME_PRECISE:
734 case CLOCK_REALTIME_FAST:
735 case CLOCK_MONOTONIC:
736 case CLOCK_MONOTONIC_PRECISE:
737 case CLOCK_MONOTONIC_FAST:
738 case CLOCK_UPTIME:
739 case CLOCK_UPTIME_PRECISE:
740 case CLOCK_UPTIME_FAST:
741 case CLOCK_SECOND:
742 timespec2bintime(&timo->end, &bt);
743 switch (timo->clockid) {
744 case CLOCK_REALTIME:
745 case CLOCK_REALTIME_PRECISE:
746 case CLOCK_REALTIME_FAST:
747 case CLOCK_SECOND:
748 getboottimebin(&bbt);
749 bintime_sub(&bt, &bbt);
750 break;
751 }
752 if (bt.sec < 0)
753 return (ETIMEDOUT);
754 if (bt.sec >= (SBT_MAX >> 32)) {
755 *sbt = 0;
756 *flags = 0;
757 return (0);
758 }
759 *sbt = bttosbt(bt);
760 umtx_abs_timeout_enforce_min(sbt);
761
762 /*
763 * Check if the absolute time should be aligned to
764 * avoid firing multiple timer events in non-periodic
765 * timer mode.
766 */
767 switch (timo->clockid) {
768 case CLOCK_REALTIME_FAST:
769 case CLOCK_MONOTONIC_FAST:
770 case CLOCK_UPTIME_FAST:
771 rem = *sbt % tc_tick_sbt;
772 if (__predict_true(rem != 0))
773 *sbt += tc_tick_sbt - rem;
774 break;
775 case CLOCK_SECOND:
776 rem = *sbt % SBT_1S;
777 if (__predict_true(rem != 0))
778 *sbt += SBT_1S - rem;
779 break;
780 }
781 *flags = C_ABSOLUTE;
782 return (0);
783
784 /* Clocks that has to be periodically polled. */
785 case CLOCK_VIRTUAL:
786 case CLOCK_PROF:
787 case CLOCK_THREAD_CPUTIME_ID:
788 case CLOCK_PROCESS_CPUTIME_ID:
789 default:
790 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
791 if (timespeccmp(&timo->end, &timo->cur, <=))
792 return (ETIMEDOUT);
793 timespecsub(&timo->end, &timo->cur, &tts);
794 *sbt = tick_sbt * tstohz(&tts);
795 *flags = C_HARDCLOCK;
796 return (0);
797 }
798 }
799
800 static uint32_t
umtx_unlock_val(uint32_t flags,bool rb)801 umtx_unlock_val(uint32_t flags, bool rb)
802 {
803
804 if (rb)
805 return (UMUTEX_RB_OWNERDEAD);
806 else if ((flags & UMUTEX_NONCONSISTENT) != 0)
807 return (UMUTEX_RB_NOTRECOV);
808 else
809 return (UMUTEX_UNOWNED);
810
811 }
812
813 /*
814 * Put thread into sleep state, before sleeping, check if
815 * thread was removed from umtx queue.
816 */
817 int
umtxq_sleep(struct umtx_q * uq,const char * wmesg,struct umtx_abs_timeout * timo)818 umtxq_sleep(struct umtx_q *uq, const char *wmesg,
819 struct umtx_abs_timeout *timo)
820 {
821 struct umtxq_chain *uc;
822 sbintime_t sbt = 0;
823 int error, flags = 0;
824
825 uc = umtxq_getchain(&uq->uq_key);
826 UMTXQ_LOCKED_ASSERT(uc);
827 for (;;) {
828 if (!(uq->uq_flags & UQF_UMTXQ)) {
829 error = 0;
830 break;
831 }
832 if (timo != NULL) {
833 if (timo->is_abs_real)
834 curthread->td_rtcgen =
835 atomic_load_acq_int(&rtc_generation);
836 error = umtx_abs_timeout_getsbt(timo, &sbt, &flags);
837 if (error != 0)
838 break;
839 }
840 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg,
841 sbt, 0, flags);
842 uc = umtxq_getchain(&uq->uq_key);
843 mtx_lock(&uc->uc_lock);
844 if (error == EINTR || error == ERESTART)
845 break;
846 if (error == EWOULDBLOCK && (flags & C_ABSOLUTE) != 0) {
847 error = ETIMEDOUT;
848 break;
849 }
850 }
851
852 curthread->td_rtcgen = 0;
853 return (error);
854 }
855
856 /*
857 * Convert userspace address into unique logical address.
858 */
859 int
umtx_key_get(const void * addr,int type,int share,struct umtx_key * key)860 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key)
861 {
862 struct thread *td = curthread;
863 vm_map_t map;
864 vm_map_entry_t entry;
865 vm_pindex_t pindex;
866 vm_prot_t prot;
867 boolean_t wired;
868
869 key->type = type;
870 if (share == THREAD_SHARE) {
871 key->shared = 0;
872 key->info.private.vs = td->td_proc->p_vmspace;
873 key->info.private.addr = (uintptr_t)addr;
874 } else {
875 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
876 map = &td->td_proc->p_vmspace->vm_map;
877 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
878 &entry, &key->info.shared.object, &pindex, &prot,
879 &wired) != KERN_SUCCESS) {
880 return (EFAULT);
881 }
882
883 if ((share == PROCESS_SHARE) ||
884 (share == AUTO_SHARE &&
885 VM_INHERIT_SHARE == entry->inheritance)) {
886 key->shared = 1;
887 key->info.shared.offset = (vm_offset_t)addr -
888 entry->start + entry->offset;
889 vm_object_reference(key->info.shared.object);
890 } else {
891 key->shared = 0;
892 key->info.private.vs = td->td_proc->p_vmspace;
893 key->info.private.addr = (uintptr_t)addr;
894 }
895 vm_map_lookup_done(map, entry);
896 }
897
898 umtxq_hash(key);
899 return (0);
900 }
901
902 /*
903 * Release key.
904 */
905 void
umtx_key_release(struct umtx_key * key)906 umtx_key_release(struct umtx_key *key)
907 {
908 if (key->shared)
909 vm_object_deallocate(key->info.shared.object);
910 }
911
912 #ifdef COMPAT_FREEBSD10
913 /*
914 * Lock a umtx object.
915 */
916 static int
do_lock_umtx(struct thread * td,struct umtx * umtx,u_long id,const struct timespec * timeout)917 do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id,
918 const struct timespec *timeout)
919 {
920 struct umtx_abs_timeout timo;
921 struct umtx_q *uq;
922 u_long owner;
923 u_long old;
924 int error = 0;
925
926 uq = td->td_umtxq;
927 if (timeout != NULL)
928 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
929
930 /*
931 * Care must be exercised when dealing with umtx structure. It
932 * can fault on any access.
933 */
934 for (;;) {
935 /*
936 * Try the uncontested case. This should be done in userland.
937 */
938 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
939
940 /* The acquire succeeded. */
941 if (owner == UMTX_UNOWNED)
942 return (0);
943
944 /* The address was invalid. */
945 if (owner == -1)
946 return (EFAULT);
947
948 /* If no one owns it but it is contested try to acquire it. */
949 if (owner == UMTX_CONTESTED) {
950 owner = casuword(&umtx->u_owner,
951 UMTX_CONTESTED, id | UMTX_CONTESTED);
952
953 if (owner == UMTX_CONTESTED)
954 return (0);
955
956 /* The address was invalid. */
957 if (owner == -1)
958 return (EFAULT);
959
960 error = thread_check_susp(td, false);
961 if (error != 0)
962 break;
963
964 /* If this failed the lock has changed, restart. */
965 continue;
966 }
967
968 /*
969 * If we caught a signal, we have retried and now
970 * exit immediately.
971 */
972 if (error != 0)
973 break;
974
975 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK,
976 AUTO_SHARE, &uq->uq_key)) != 0)
977 return (error);
978
979 umtxq_lock(&uq->uq_key);
980 umtxq_busy(&uq->uq_key);
981 umtxq_insert(uq);
982 umtxq_unbusy(&uq->uq_key);
983 umtxq_unlock(&uq->uq_key);
984
985 /*
986 * Set the contested bit so that a release in user space
987 * knows to use the system call for unlock. If this fails
988 * either some one else has acquired the lock or it has been
989 * released.
990 */
991 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
992
993 /* The address was invalid. */
994 if (old == -1) {
995 umtxq_lock(&uq->uq_key);
996 umtxq_remove(uq);
997 umtxq_unlock(&uq->uq_key);
998 umtx_key_release(&uq->uq_key);
999 return (EFAULT);
1000 }
1001
1002 /*
1003 * We set the contested bit, sleep. Otherwise the lock changed
1004 * and we need to retry or we lost a race to the thread
1005 * unlocking the umtx.
1006 */
1007 umtxq_lock(&uq->uq_key);
1008 if (old == owner)
1009 error = umtxq_sleep(uq, "umtx", timeout == NULL ? NULL :
1010 &timo);
1011 umtxq_remove(uq);
1012 umtxq_unlock(&uq->uq_key);
1013 umtx_key_release(&uq->uq_key);
1014
1015 if (error == 0)
1016 error = thread_check_susp(td, false);
1017 }
1018
1019 if (timeout == NULL) {
1020 /* Mutex locking is restarted if it is interrupted. */
1021 if (error == EINTR)
1022 error = ERESTART;
1023 } else {
1024 /* Timed-locking is not restarted. */
1025 if (error == ERESTART)
1026 error = EINTR;
1027 }
1028 return (error);
1029 }
1030
1031 /*
1032 * Unlock a umtx object.
1033 */
1034 static int
do_unlock_umtx(struct thread * td,struct umtx * umtx,u_long id)1035 do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id)
1036 {
1037 struct umtx_key key;
1038 u_long owner;
1039 u_long old;
1040 int error;
1041 int count;
1042
1043 /*
1044 * Make sure we own this mtx.
1045 */
1046 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
1047 if (owner == -1)
1048 return (EFAULT);
1049
1050 if ((owner & ~UMTX_CONTESTED) != id)
1051 return (EPERM);
1052
1053 /* This should be done in userland */
1054 if ((owner & UMTX_CONTESTED) == 0) {
1055 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
1056 if (old == -1)
1057 return (EFAULT);
1058 if (old == owner)
1059 return (0);
1060 owner = old;
1061 }
1062
1063 /* We should only ever be in here for contested locks */
1064 if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1065 &key)) != 0)
1066 return (error);
1067
1068 umtxq_lock(&key);
1069 umtxq_busy(&key);
1070 count = umtxq_count(&key);
1071 umtxq_unlock(&key);
1072
1073 /*
1074 * When unlocking the umtx, it must be marked as unowned if
1075 * there is zero or one thread only waiting for it.
1076 * Otherwise, it must be marked as contested.
1077 */
1078 old = casuword(&umtx->u_owner, owner,
1079 count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED);
1080 umtxq_lock(&key);
1081 umtxq_signal(&key,1);
1082 umtxq_unbusy(&key);
1083 umtxq_unlock(&key);
1084 umtx_key_release(&key);
1085 if (old == -1)
1086 return (EFAULT);
1087 if (old != owner)
1088 return (EINVAL);
1089 return (0);
1090 }
1091
1092 #ifdef COMPAT_FREEBSD32
1093
1094 /*
1095 * Lock a umtx object.
1096 */
1097 static int
do_lock_umtx32(struct thread * td,uint32_t * m,uint32_t id,const struct timespec * timeout)1098 do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id,
1099 const struct timespec *timeout)
1100 {
1101 struct umtx_abs_timeout timo;
1102 struct umtx_q *uq;
1103 uint32_t owner;
1104 uint32_t old;
1105 int error = 0;
1106
1107 uq = td->td_umtxq;
1108
1109 if (timeout != NULL)
1110 umtx_abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout);
1111
1112 /*
1113 * Care must be exercised when dealing with umtx structure. It
1114 * can fault on any access.
1115 */
1116 for (;;) {
1117 /*
1118 * Try the uncontested case. This should be done in userland.
1119 */
1120 owner = casuword32(m, UMUTEX_UNOWNED, id);
1121
1122 /* The acquire succeeded. */
1123 if (owner == UMUTEX_UNOWNED)
1124 return (0);
1125
1126 /* The address was invalid. */
1127 if (owner == -1)
1128 return (EFAULT);
1129
1130 /* If no one owns it but it is contested try to acquire it. */
1131 if (owner == UMUTEX_CONTESTED) {
1132 owner = casuword32(m,
1133 UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
1134 if (owner == UMUTEX_CONTESTED)
1135 return (0);
1136
1137 /* The address was invalid. */
1138 if (owner == -1)
1139 return (EFAULT);
1140
1141 error = thread_check_susp(td, false);
1142 if (error != 0)
1143 break;
1144
1145 /* If this failed the lock has changed, restart. */
1146 continue;
1147 }
1148
1149 /*
1150 * If we caught a signal, we have retried and now
1151 * exit immediately.
1152 */
1153 if (error != 0)
1154 return (error);
1155
1156 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK,
1157 AUTO_SHARE, &uq->uq_key)) != 0)
1158 return (error);
1159
1160 umtxq_lock(&uq->uq_key);
1161 umtxq_busy(&uq->uq_key);
1162 umtxq_insert(uq);
1163 umtxq_unbusy(&uq->uq_key);
1164 umtxq_unlock(&uq->uq_key);
1165
1166 /*
1167 * Set the contested bit so that a release in user space
1168 * knows to use the system call for unlock. If this fails
1169 * either some one else has acquired the lock or it has been
1170 * released.
1171 */
1172 old = casuword32(m, owner, owner | UMUTEX_CONTESTED);
1173
1174 /* The address was invalid. */
1175 if (old == -1) {
1176 umtxq_lock(&uq->uq_key);
1177 umtxq_remove(uq);
1178 umtxq_unlock(&uq->uq_key);
1179 umtx_key_release(&uq->uq_key);
1180 return (EFAULT);
1181 }
1182
1183 /*
1184 * We set the contested bit, sleep. Otherwise the lock changed
1185 * and we need to retry or we lost a race to the thread
1186 * unlocking the umtx.
1187 */
1188 umtxq_lock(&uq->uq_key);
1189 if (old == owner)
1190 error = umtxq_sleep(uq, "umtx", timeout == NULL ?
1191 NULL : &timo);
1192 umtxq_remove(uq);
1193 umtxq_unlock(&uq->uq_key);
1194 umtx_key_release(&uq->uq_key);
1195
1196 if (error == 0)
1197 error = thread_check_susp(td, false);
1198 }
1199
1200 if (timeout == NULL) {
1201 /* Mutex locking is restarted if it is interrupted. */
1202 if (error == EINTR)
1203 error = ERESTART;
1204 } else {
1205 /* Timed-locking is not restarted. */
1206 if (error == ERESTART)
1207 error = EINTR;
1208 }
1209 return (error);
1210 }
1211
1212 /*
1213 * Unlock a umtx object.
1214 */
1215 static int
do_unlock_umtx32(struct thread * td,uint32_t * m,uint32_t id)1216 do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id)
1217 {
1218 struct umtx_key key;
1219 uint32_t owner;
1220 uint32_t old;
1221 int error;
1222 int count;
1223
1224 /*
1225 * Make sure we own this mtx.
1226 */
1227 owner = fuword32(m);
1228 if (owner == -1)
1229 return (EFAULT);
1230
1231 if ((owner & ~UMUTEX_CONTESTED) != id)
1232 return (EPERM);
1233
1234 /* This should be done in userland */
1235 if ((owner & UMUTEX_CONTESTED) == 0) {
1236 old = casuword32(m, owner, UMUTEX_UNOWNED);
1237 if (old == -1)
1238 return (EFAULT);
1239 if (old == owner)
1240 return (0);
1241 owner = old;
1242 }
1243
1244 /* We should only ever be in here for contested locks */
1245 if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE,
1246 &key)) != 0)
1247 return (error);
1248
1249 umtxq_lock(&key);
1250 umtxq_busy(&key);
1251 count = umtxq_count(&key);
1252 umtxq_unlock(&key);
1253
1254 /*
1255 * When unlocking the umtx, it must be marked as unowned if
1256 * there is zero or one thread only waiting for it.
1257 * Otherwise, it must be marked as contested.
1258 */
1259 old = casuword32(m, owner,
1260 count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
1261 umtxq_lock(&key);
1262 umtxq_signal(&key,1);
1263 umtxq_unbusy(&key);
1264 umtxq_unlock(&key);
1265 umtx_key_release(&key);
1266 if (old == -1)
1267 return (EFAULT);
1268 if (old != owner)
1269 return (EINVAL);
1270 return (0);
1271 }
1272 #endif /* COMPAT_FREEBSD32 */
1273 #endif /* COMPAT_FREEBSD10 */
1274
1275 /*
1276 * Fetch and compare value, sleep on the address if value is not changed.
1277 */
1278 static int
do_wait(struct thread * td,void * addr,u_long id,struct _umtx_time * timeout,int compat32,int is_private)1279 do_wait(struct thread *td, void *addr, u_long id,
1280 struct _umtx_time *timeout, int compat32, int is_private)
1281 {
1282 struct umtx_abs_timeout timo;
1283 struct umtx_q *uq;
1284 u_long tmp;
1285 uint32_t tmp32;
1286 int error = 0;
1287
1288 uq = td->td_umtxq;
1289 if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
1290 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1291 return (error);
1292
1293 if (timeout != NULL)
1294 umtx_abs_timeout_init2(&timo, timeout);
1295
1296 umtxq_lock(&uq->uq_key);
1297 umtxq_insert(uq);
1298 umtxq_unlock(&uq->uq_key);
1299 if (compat32 == 0) {
1300 error = fueword(addr, &tmp);
1301 if (error != 0)
1302 error = EFAULT;
1303 } else {
1304 error = fueword32(addr, &tmp32);
1305 if (error == 0)
1306 tmp = tmp32;
1307 else
1308 error = EFAULT;
1309 }
1310 umtxq_lock(&uq->uq_key);
1311 if (error == 0) {
1312 if (tmp == id)
1313 error = umtxq_sleep(uq, "uwait", timeout == NULL ?
1314 NULL : &timo);
1315 if ((uq->uq_flags & UQF_UMTXQ) == 0)
1316 error = 0;
1317 else
1318 umtxq_remove(uq);
1319 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
1320 umtxq_remove(uq);
1321 }
1322 umtxq_unlock(&uq->uq_key);
1323 umtx_key_release(&uq->uq_key);
1324 if (error == ERESTART)
1325 error = EINTR;
1326 return (error);
1327 }
1328
1329 /*
1330 * Wake up threads sleeping on the specified address.
1331 */
1332 int
kern_umtx_wake(struct thread * td,void * uaddr,int n_wake,int is_private)1333 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
1334 {
1335 struct umtx_key key;
1336 int ret;
1337
1338 if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
1339 is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
1340 return (ret);
1341 umtxq_lock(&key);
1342 umtxq_signal(&key, n_wake);
1343 umtxq_unlock(&key);
1344 umtx_key_release(&key);
1345 return (0);
1346 }
1347
1348 /*
1349 * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
1350 */
1351 static int
do_lock_normal(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int mode)1352 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
1353 struct _umtx_time *timeout, int mode)
1354 {
1355 struct umtx_abs_timeout timo;
1356 struct umtx_q *uq;
1357 uint32_t owner, old, id;
1358 int error, rv;
1359
1360 id = td->td_tid;
1361 uq = td->td_umtxq;
1362 error = 0;
1363 if (timeout != NULL)
1364 umtx_abs_timeout_init2(&timo, timeout);
1365
1366 /*
1367 * Care must be exercised when dealing with umtx structure. It
1368 * can fault on any access.
1369 */
1370 for (;;) {
1371 rv = fueword32(&m->m_owner, &owner);
1372 if (rv == -1)
1373 return (EFAULT);
1374 if (mode == _UMUTEX_WAIT) {
1375 if (owner == UMUTEX_UNOWNED ||
1376 owner == UMUTEX_CONTESTED ||
1377 owner == UMUTEX_RB_OWNERDEAD ||
1378 owner == UMUTEX_RB_NOTRECOV)
1379 return (0);
1380 } else {
1381 /*
1382 * Robust mutex terminated. Kernel duty is to
1383 * return EOWNERDEAD to the userspace. The
1384 * umutex.m_flags UMUTEX_NONCONSISTENT is set
1385 * by the common userspace code.
1386 */
1387 if (owner == UMUTEX_RB_OWNERDEAD) {
1388 rv = casueword32(&m->m_owner,
1389 UMUTEX_RB_OWNERDEAD, &owner,
1390 id | UMUTEX_CONTESTED);
1391 if (rv == -1)
1392 return (EFAULT);
1393 if (rv == 0) {
1394 MPASS(owner == UMUTEX_RB_OWNERDEAD);
1395 return (EOWNERDEAD); /* success */
1396 }
1397 MPASS(rv == 1);
1398 rv = thread_check_susp(td, false);
1399 if (rv != 0)
1400 return (rv);
1401 continue;
1402 }
1403 if (owner == UMUTEX_RB_NOTRECOV)
1404 return (ENOTRECOVERABLE);
1405
1406 /*
1407 * Try the uncontested case. This should be
1408 * done in userland.
1409 */
1410 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
1411 &owner, id);
1412 /* The address was invalid. */
1413 if (rv == -1)
1414 return (EFAULT);
1415
1416 /* The acquire succeeded. */
1417 if (rv == 0) {
1418 MPASS(owner == UMUTEX_UNOWNED);
1419 return (0);
1420 }
1421
1422 /*
1423 * If no one owns it but it is contested try
1424 * to acquire it.
1425 */
1426 MPASS(rv == 1);
1427 if (owner == UMUTEX_CONTESTED) {
1428 rv = casueword32(&m->m_owner,
1429 UMUTEX_CONTESTED, &owner,
1430 id | UMUTEX_CONTESTED);
1431 /* The address was invalid. */
1432 if (rv == -1)
1433 return (EFAULT);
1434 if (rv == 0) {
1435 MPASS(owner == UMUTEX_CONTESTED);
1436 return (0);
1437 }
1438 if (rv == 1) {
1439 rv = thread_check_susp(td, false);
1440 if (rv != 0)
1441 return (rv);
1442 }
1443
1444 /*
1445 * If this failed the lock has
1446 * changed, restart.
1447 */
1448 continue;
1449 }
1450
1451 /* rv == 1 but not contested, likely store failure */
1452 rv = thread_check_susp(td, false);
1453 if (rv != 0)
1454 return (rv);
1455 }
1456
1457 if (mode == _UMUTEX_TRY)
1458 return (EBUSY);
1459
1460 /*
1461 * If we caught a signal, we have retried and now
1462 * exit immediately.
1463 */
1464 if (error != 0)
1465 return (error);
1466
1467 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1468 GET_SHARE(flags), &uq->uq_key)) != 0)
1469 return (error);
1470
1471 umtxq_lock(&uq->uq_key);
1472 umtxq_busy(&uq->uq_key);
1473 umtxq_insert(uq);
1474 umtxq_unlock(&uq->uq_key);
1475
1476 /*
1477 * Set the contested bit so that a release in user space
1478 * knows to use the system call for unlock. If this fails
1479 * either some one else has acquired the lock or it has been
1480 * released.
1481 */
1482 rv = casueword32(&m->m_owner, owner, &old,
1483 owner | UMUTEX_CONTESTED);
1484
1485 /* The address was invalid or casueword failed to store. */
1486 if (rv == -1 || rv == 1) {
1487 umtxq_lock(&uq->uq_key);
1488 umtxq_remove(uq);
1489 umtxq_unbusy(&uq->uq_key);
1490 umtxq_unlock(&uq->uq_key);
1491 umtx_key_release(&uq->uq_key);
1492 if (rv == -1)
1493 return (EFAULT);
1494 if (rv == 1) {
1495 rv = thread_check_susp(td, false);
1496 if (rv != 0)
1497 return (rv);
1498 }
1499 continue;
1500 }
1501
1502 /*
1503 * We set the contested bit, sleep. Otherwise the lock changed
1504 * and we need to retry or we lost a race to the thread
1505 * unlocking the umtx.
1506 */
1507 umtxq_lock(&uq->uq_key);
1508 umtxq_unbusy(&uq->uq_key);
1509 MPASS(old == owner);
1510 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1511 NULL : &timo);
1512 umtxq_remove(uq);
1513 umtxq_unlock(&uq->uq_key);
1514 umtx_key_release(&uq->uq_key);
1515
1516 if (error == 0)
1517 error = thread_check_susp(td, false);
1518 }
1519
1520 return (0);
1521 }
1522
1523 /*
1524 * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1525 */
1526 static int
do_unlock_normal(struct thread * td,struct umutex * m,uint32_t flags,bool rb)1527 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
1528 {
1529 struct umtx_key key;
1530 uint32_t owner, old, id, newlock;
1531 int error, count;
1532
1533 id = td->td_tid;
1534
1535 again:
1536 /*
1537 * Make sure we own this mtx.
1538 */
1539 error = fueword32(&m->m_owner, &owner);
1540 if (error == -1)
1541 return (EFAULT);
1542
1543 if ((owner & ~UMUTEX_CONTESTED) != id)
1544 return (EPERM);
1545
1546 newlock = umtx_unlock_val(flags, rb);
1547 if ((owner & UMUTEX_CONTESTED) == 0) {
1548 error = casueword32(&m->m_owner, owner, &old, newlock);
1549 if (error == -1)
1550 return (EFAULT);
1551 if (error == 1) {
1552 error = thread_check_susp(td, false);
1553 if (error != 0)
1554 return (error);
1555 goto again;
1556 }
1557 MPASS(old == owner);
1558 return (0);
1559 }
1560
1561 /* We should only ever be in here for contested locks */
1562 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1563 &key)) != 0)
1564 return (error);
1565
1566 umtxq_lock(&key);
1567 umtxq_busy(&key);
1568 count = umtxq_count(&key);
1569 umtxq_unlock(&key);
1570
1571 /*
1572 * When unlocking the umtx, it must be marked as unowned if
1573 * there is zero or one thread only waiting for it.
1574 * Otherwise, it must be marked as contested.
1575 */
1576 if (count > 1)
1577 newlock |= UMUTEX_CONTESTED;
1578 error = casueword32(&m->m_owner, owner, &old, newlock);
1579 umtxq_lock(&key);
1580 umtxq_signal(&key, 1);
1581 umtxq_unbusy(&key);
1582 umtxq_unlock(&key);
1583 umtx_key_release(&key);
1584 if (error == -1)
1585 return (EFAULT);
1586 if (error == 1) {
1587 if (old != owner)
1588 return (EINVAL);
1589 error = thread_check_susp(td, false);
1590 if (error != 0)
1591 return (error);
1592 goto again;
1593 }
1594 return (0);
1595 }
1596
1597 /*
1598 * Check if the mutex is available and wake up a waiter,
1599 * only for simple mutex.
1600 */
1601 static int
do_wake_umutex(struct thread * td,struct umutex * m)1602 do_wake_umutex(struct thread *td, struct umutex *m)
1603 {
1604 struct umtx_key key;
1605 uint32_t owner;
1606 uint32_t flags;
1607 int error;
1608 int count;
1609
1610 again:
1611 error = fueword32(&m->m_owner, &owner);
1612 if (error == -1)
1613 return (EFAULT);
1614
1615 if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD &&
1616 owner != UMUTEX_RB_NOTRECOV)
1617 return (0);
1618
1619 error = fueword32(&m->m_flags, &flags);
1620 if (error == -1)
1621 return (EFAULT);
1622
1623 /* We should only ever be in here for contested locks */
1624 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1625 &key)) != 0)
1626 return (error);
1627
1628 umtxq_lock(&key);
1629 umtxq_busy(&key);
1630 count = umtxq_count(&key);
1631 umtxq_unlock(&key);
1632
1633 if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD &&
1634 owner != UMUTEX_RB_NOTRECOV) {
1635 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
1636 UMUTEX_UNOWNED);
1637 if (error == -1) {
1638 error = EFAULT;
1639 } else if (error == 1) {
1640 umtxq_lock(&key);
1641 umtxq_unbusy(&key);
1642 umtxq_unlock(&key);
1643 umtx_key_release(&key);
1644 error = thread_check_susp(td, false);
1645 if (error != 0)
1646 return (error);
1647 goto again;
1648 }
1649 }
1650
1651 umtxq_lock(&key);
1652 if (error == 0 && count != 0) {
1653 MPASS((owner & ~UMUTEX_CONTESTED) == 0 ||
1654 owner == UMUTEX_RB_OWNERDEAD ||
1655 owner == UMUTEX_RB_NOTRECOV);
1656 umtxq_signal(&key, 1);
1657 }
1658 umtxq_unbusy(&key);
1659 umtxq_unlock(&key);
1660 umtx_key_release(&key);
1661 return (error);
1662 }
1663
1664 /*
1665 * Check if the mutex has waiters and tries to fix contention bit.
1666 */
1667 static int
do_wake2_umutex(struct thread * td,struct umutex * m,uint32_t flags)1668 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
1669 {
1670 struct umtx_key key;
1671 uint32_t owner, old;
1672 int type;
1673 int error;
1674 int count;
1675
1676 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT |
1677 UMUTEX_ROBUST)) {
1678 case 0:
1679 case UMUTEX_ROBUST:
1680 type = TYPE_NORMAL_UMUTEX;
1681 break;
1682 case UMUTEX_PRIO_INHERIT:
1683 type = TYPE_PI_UMUTEX;
1684 break;
1685 case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST):
1686 type = TYPE_PI_ROBUST_UMUTEX;
1687 break;
1688 case UMUTEX_PRIO_PROTECT:
1689 type = TYPE_PP_UMUTEX;
1690 break;
1691 case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST):
1692 type = TYPE_PP_ROBUST_UMUTEX;
1693 break;
1694 default:
1695 return (EINVAL);
1696 }
1697 if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0)
1698 return (error);
1699
1700 owner = 0;
1701 umtxq_lock(&key);
1702 umtxq_busy(&key);
1703 count = umtxq_count(&key);
1704 umtxq_unlock(&key);
1705
1706 error = fueword32(&m->m_owner, &owner);
1707 if (error == -1)
1708 error = EFAULT;
1709
1710 /*
1711 * Only repair contention bit if there is a waiter, this means
1712 * the mutex is still being referenced by userland code,
1713 * otherwise don't update any memory.
1714 */
1715 while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 &&
1716 (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) {
1717 error = casueword32(&m->m_owner, owner, &old,
1718 owner | UMUTEX_CONTESTED);
1719 if (error == -1) {
1720 error = EFAULT;
1721 break;
1722 }
1723 if (error == 0) {
1724 MPASS(old == owner);
1725 break;
1726 }
1727 owner = old;
1728 error = thread_check_susp(td, false);
1729 }
1730
1731 umtxq_lock(&key);
1732 if (error == EFAULT) {
1733 umtxq_signal(&key, INT_MAX);
1734 } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
1735 owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
1736 umtxq_signal(&key, 1);
1737 umtxq_unbusy(&key);
1738 umtxq_unlock(&key);
1739 umtx_key_release(&key);
1740 return (error);
1741 }
1742
1743 struct umtx_pi *
umtx_pi_alloc(int flags)1744 umtx_pi_alloc(int flags)
1745 {
1746 struct umtx_pi *pi;
1747
1748 pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1749 if (pi == NULL)
1750 return (NULL);
1751
1752 TAILQ_INIT(&pi->pi_blocked);
1753 atomic_add_int(&umtx_pi_allocated, 1);
1754 return (pi);
1755 }
1756
1757 void
umtx_pi_free(struct umtx_pi * pi)1758 umtx_pi_free(struct umtx_pi *pi)
1759 {
1760 uma_zfree(umtx_pi_zone, pi);
1761 atomic_add_int(&umtx_pi_allocated, -1);
1762 }
1763
1764 /*
1765 * Adjust the thread's position on a pi_state after its priority has been
1766 * changed.
1767 */
1768 static int
umtx_pi_adjust_thread(struct umtx_pi * pi,struct thread * td)1769 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1770 {
1771 struct umtx_q *uq, *uq1, *uq2;
1772 struct thread *td1;
1773
1774 mtx_assert(&umtx_lock, MA_OWNED);
1775 if (pi == NULL)
1776 return (0);
1777
1778 uq = td->td_umtxq;
1779
1780 /*
1781 * Check if the thread needs to be moved on the blocked chain.
1782 * It needs to be moved if either its priority is lower than
1783 * the previous thread or higher than the next thread.
1784 */
1785 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1786 uq2 = TAILQ_NEXT(uq, uq_lockq);
1787 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1788 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1789 /*
1790 * Remove thread from blocked chain and determine where
1791 * it should be moved to.
1792 */
1793 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1794 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1795 td1 = uq1->uq_thread;
1796 MPASS(td1->td_proc->p_magic == P_MAGIC);
1797 if (UPRI(td1) > UPRI(td))
1798 break;
1799 }
1800
1801 if (uq1 == NULL)
1802 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1803 else
1804 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1805 }
1806 return (1);
1807 }
1808
1809 static struct umtx_pi *
umtx_pi_next(struct umtx_pi * pi)1810 umtx_pi_next(struct umtx_pi *pi)
1811 {
1812 struct umtx_q *uq_owner;
1813
1814 if (pi->pi_owner == NULL)
1815 return (NULL);
1816 uq_owner = pi->pi_owner->td_umtxq;
1817 if (uq_owner == NULL)
1818 return (NULL);
1819 return (uq_owner->uq_pi_blocked);
1820 }
1821
1822 /*
1823 * Floyd's Cycle-Finding Algorithm.
1824 */
1825 static bool
umtx_pi_check_loop(struct umtx_pi * pi)1826 umtx_pi_check_loop(struct umtx_pi *pi)
1827 {
1828 struct umtx_pi *pi1; /* fast iterator */
1829
1830 mtx_assert(&umtx_lock, MA_OWNED);
1831 if (pi == NULL)
1832 return (false);
1833 pi1 = pi;
1834 for (;;) {
1835 pi = umtx_pi_next(pi);
1836 if (pi == NULL)
1837 break;
1838 pi1 = umtx_pi_next(pi1);
1839 if (pi1 == NULL)
1840 break;
1841 pi1 = umtx_pi_next(pi1);
1842 if (pi1 == NULL)
1843 break;
1844 if (pi == pi1)
1845 return (true);
1846 }
1847 return (false);
1848 }
1849
1850 /*
1851 * Propagate priority when a thread is blocked on POSIX
1852 * PI mutex.
1853 */
1854 static void
umtx_propagate_priority(struct thread * td)1855 umtx_propagate_priority(struct thread *td)
1856 {
1857 struct umtx_q *uq;
1858 struct umtx_pi *pi;
1859 int pri;
1860
1861 mtx_assert(&umtx_lock, MA_OWNED);
1862 pri = UPRI(td);
1863 uq = td->td_umtxq;
1864 pi = uq->uq_pi_blocked;
1865 if (pi == NULL)
1866 return;
1867 if (umtx_pi_check_loop(pi))
1868 return;
1869
1870 for (;;) {
1871 td = pi->pi_owner;
1872 if (td == NULL || td == curthread)
1873 return;
1874
1875 MPASS(td->td_proc != NULL);
1876 MPASS(td->td_proc->p_magic == P_MAGIC);
1877
1878 thread_lock(td);
1879 if (td->td_lend_user_pri > pri)
1880 sched_lend_user_prio(td, pri);
1881 else {
1882 thread_unlock(td);
1883 break;
1884 }
1885 thread_unlock(td);
1886
1887 /*
1888 * Pick up the lock that td is blocked on.
1889 */
1890 uq = td->td_umtxq;
1891 pi = uq->uq_pi_blocked;
1892 if (pi == NULL)
1893 break;
1894 /* Resort td on the list if needed. */
1895 umtx_pi_adjust_thread(pi, td);
1896 }
1897 }
1898
1899 /*
1900 * Unpropagate priority for a PI mutex when a thread blocked on
1901 * it is interrupted by signal or resumed by others.
1902 */
1903 static void
umtx_repropagate_priority(struct umtx_pi * pi)1904 umtx_repropagate_priority(struct umtx_pi *pi)
1905 {
1906 struct umtx_q *uq, *uq_owner;
1907 struct umtx_pi *pi2;
1908 int pri;
1909
1910 mtx_assert(&umtx_lock, MA_OWNED);
1911
1912 if (umtx_pi_check_loop(pi))
1913 return;
1914 while (pi != NULL && pi->pi_owner != NULL) {
1915 pri = PRI_MAX;
1916 uq_owner = pi->pi_owner->td_umtxq;
1917
1918 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1919 uq = TAILQ_FIRST(&pi2->pi_blocked);
1920 if (uq != NULL) {
1921 if (pri > UPRI(uq->uq_thread))
1922 pri = UPRI(uq->uq_thread);
1923 }
1924 }
1925
1926 if (pri > uq_owner->uq_inherited_pri)
1927 pri = uq_owner->uq_inherited_pri;
1928 thread_lock(pi->pi_owner);
1929 sched_lend_user_prio(pi->pi_owner, pri);
1930 thread_unlock(pi->pi_owner);
1931 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1932 umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1933 }
1934 }
1935
1936 /*
1937 * Insert a PI mutex into owned list.
1938 */
1939 static void
umtx_pi_setowner(struct umtx_pi * pi,struct thread * owner)1940 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1941 {
1942 struct umtx_q *uq_owner;
1943
1944 uq_owner = owner->td_umtxq;
1945 mtx_assert(&umtx_lock, MA_OWNED);
1946 MPASS(pi->pi_owner == NULL);
1947 pi->pi_owner = owner;
1948 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1949 }
1950
1951 /*
1952 * Disown a PI mutex, and remove it from the owned list.
1953 */
1954 static void
umtx_pi_disown(struct umtx_pi * pi)1955 umtx_pi_disown(struct umtx_pi *pi)
1956 {
1957
1958 mtx_assert(&umtx_lock, MA_OWNED);
1959 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
1960 pi->pi_owner = NULL;
1961 }
1962
1963 /*
1964 * Claim ownership of a PI mutex.
1965 */
1966 int
umtx_pi_claim(struct umtx_pi * pi,struct thread * owner)1967 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1968 {
1969 struct umtx_q *uq;
1970 int pri;
1971
1972 mtx_lock(&umtx_lock);
1973 if (pi->pi_owner == owner) {
1974 mtx_unlock(&umtx_lock);
1975 return (0);
1976 }
1977
1978 if (pi->pi_owner != NULL) {
1979 /*
1980 * userland may have already messed the mutex, sigh.
1981 */
1982 mtx_unlock(&umtx_lock);
1983 return (EPERM);
1984 }
1985 umtx_pi_setowner(pi, owner);
1986 uq = TAILQ_FIRST(&pi->pi_blocked);
1987 if (uq != NULL) {
1988 pri = UPRI(uq->uq_thread);
1989 thread_lock(owner);
1990 if (pri < UPRI(owner))
1991 sched_lend_user_prio(owner, pri);
1992 thread_unlock(owner);
1993 }
1994 mtx_unlock(&umtx_lock);
1995 return (0);
1996 }
1997
1998 /*
1999 * Adjust a thread's order position in its blocked PI mutex,
2000 * this may result new priority propagating process.
2001 */
2002 void
umtx_pi_adjust(struct thread * td,u_char oldpri)2003 umtx_pi_adjust(struct thread *td, u_char oldpri)
2004 {
2005 struct umtx_q *uq;
2006 struct umtx_pi *pi;
2007
2008 uq = td->td_umtxq;
2009 mtx_lock(&umtx_lock);
2010 /*
2011 * Pick up the lock that td is blocked on.
2012 */
2013 pi = uq->uq_pi_blocked;
2014 if (pi != NULL) {
2015 umtx_pi_adjust_thread(pi, td);
2016 umtx_repropagate_priority(pi);
2017 }
2018 mtx_unlock(&umtx_lock);
2019 }
2020
2021 /*
2022 * Sleep on a PI mutex.
2023 */
2024 int
umtxq_sleep_pi(struct umtx_q * uq,struct umtx_pi * pi,uint32_t owner,const char * wmesg,struct umtx_abs_timeout * timo,bool shared)2025 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
2026 const char *wmesg, struct umtx_abs_timeout *timo, bool shared)
2027 {
2028 struct thread *td, *td1;
2029 struct umtx_q *uq1;
2030 int error, pri;
2031 #ifdef INVARIANTS
2032 struct umtxq_chain *uc;
2033
2034 uc = umtxq_getchain(&pi->pi_key);
2035 #endif
2036 error = 0;
2037 td = uq->uq_thread;
2038 KASSERT(td == curthread, ("inconsistent uq_thread"));
2039 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
2040 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
2041 umtxq_insert(uq);
2042 mtx_lock(&umtx_lock);
2043 if (pi->pi_owner == NULL) {
2044 mtx_unlock(&umtx_lock);
2045 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
2046 mtx_lock(&umtx_lock);
2047 if (td1 != NULL) {
2048 if (pi->pi_owner == NULL)
2049 umtx_pi_setowner(pi, td1);
2050 PROC_UNLOCK(td1->td_proc);
2051 }
2052 }
2053
2054 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
2055 pri = UPRI(uq1->uq_thread);
2056 if (pri > UPRI(td))
2057 break;
2058 }
2059
2060 if (uq1 != NULL)
2061 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
2062 else
2063 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
2064
2065 uq->uq_pi_blocked = pi;
2066 thread_lock(td);
2067 td->td_flags |= TDF_UPIBLOCKED;
2068 thread_unlock(td);
2069 umtx_propagate_priority(td);
2070 mtx_unlock(&umtx_lock);
2071 umtxq_unbusy(&uq->uq_key);
2072
2073 error = umtxq_sleep(uq, wmesg, timo);
2074 umtxq_remove(uq);
2075
2076 mtx_lock(&umtx_lock);
2077 uq->uq_pi_blocked = NULL;
2078 thread_lock(td);
2079 td->td_flags &= ~TDF_UPIBLOCKED;
2080 thread_unlock(td);
2081 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
2082 umtx_repropagate_priority(pi);
2083 mtx_unlock(&umtx_lock);
2084 umtxq_unlock(&uq->uq_key);
2085
2086 return (error);
2087 }
2088
2089 /*
2090 * Add reference count for a PI mutex.
2091 */
2092 void
umtx_pi_ref(struct umtx_pi * pi)2093 umtx_pi_ref(struct umtx_pi *pi)
2094 {
2095
2096 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key));
2097 pi->pi_refcount++;
2098 }
2099
2100 /*
2101 * Decrease reference count for a PI mutex, if the counter
2102 * is decreased to zero, its memory space is freed.
2103 */
2104 void
umtx_pi_unref(struct umtx_pi * pi)2105 umtx_pi_unref(struct umtx_pi *pi)
2106 {
2107 struct umtxq_chain *uc;
2108
2109 uc = umtxq_getchain(&pi->pi_key);
2110 UMTXQ_LOCKED_ASSERT(uc);
2111 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
2112 if (--pi->pi_refcount == 0) {
2113 mtx_lock(&umtx_lock);
2114 if (pi->pi_owner != NULL)
2115 umtx_pi_disown(pi);
2116 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
2117 ("blocked queue not empty"));
2118 mtx_unlock(&umtx_lock);
2119 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
2120 umtx_pi_free(pi);
2121 }
2122 }
2123
2124 /*
2125 * Find a PI mutex in hash table.
2126 */
2127 struct umtx_pi *
umtx_pi_lookup(struct umtx_key * key)2128 umtx_pi_lookup(struct umtx_key *key)
2129 {
2130 struct umtxq_chain *uc;
2131 struct umtx_pi *pi;
2132
2133 uc = umtxq_getchain(key);
2134 UMTXQ_LOCKED_ASSERT(uc);
2135
2136 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
2137 if (umtx_key_match(&pi->pi_key, key)) {
2138 return (pi);
2139 }
2140 }
2141 return (NULL);
2142 }
2143
2144 /*
2145 * Insert a PI mutex into hash table.
2146 */
2147 void
umtx_pi_insert(struct umtx_pi * pi)2148 umtx_pi_insert(struct umtx_pi *pi)
2149 {
2150 struct umtxq_chain *uc;
2151
2152 uc = umtxq_getchain(&pi->pi_key);
2153 UMTXQ_LOCKED_ASSERT(uc);
2154 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
2155 }
2156
2157 /*
2158 * Drop a PI mutex and wakeup a top waiter.
2159 */
2160 int
umtx_pi_drop(struct thread * td,struct umtx_key * key,bool rb,int * count)2161 umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count)
2162 {
2163 struct umtx_q *uq_first, *uq_first2, *uq_me;
2164 struct umtx_pi *pi, *pi2;
2165 int pri;
2166
2167 UMTXQ_ASSERT_LOCKED_BUSY(key);
2168 *count = umtxq_count_pi(key, &uq_first);
2169 if (uq_first != NULL) {
2170 mtx_lock(&umtx_lock);
2171 pi = uq_first->uq_pi_blocked;
2172 KASSERT(pi != NULL, ("pi == NULL?"));
2173 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
2174 mtx_unlock(&umtx_lock);
2175 /* userland messed the mutex */
2176 return (EPERM);
2177 }
2178 uq_me = td->td_umtxq;
2179 if (pi->pi_owner == td)
2180 umtx_pi_disown(pi);
2181 /* get highest priority thread which is still sleeping. */
2182 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2183 while (uq_first != NULL &&
2184 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2185 uq_first = TAILQ_NEXT(uq_first, uq_lockq);
2186 }
2187 pri = PRI_MAX;
2188 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2189 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2190 if (uq_first2 != NULL) {
2191 if (pri > UPRI(uq_first2->uq_thread))
2192 pri = UPRI(uq_first2->uq_thread);
2193 }
2194 }
2195 thread_lock(td);
2196 sched_lend_user_prio(td, pri);
2197 thread_unlock(td);
2198 mtx_unlock(&umtx_lock);
2199 if (uq_first)
2200 umtxq_signal_thread(uq_first);
2201 } else {
2202 pi = umtx_pi_lookup(key);
2203 /*
2204 * A umtx_pi can exist if a signal or timeout removed the
2205 * last waiter from the umtxq, but there is still
2206 * a thread in do_lock_pi() holding the umtx_pi.
2207 */
2208 if (pi != NULL) {
2209 /*
2210 * The umtx_pi can be unowned, such as when a thread
2211 * has just entered do_lock_pi(), allocated the
2212 * umtx_pi, and unlocked the umtxq.
2213 * If the current thread owns it, it must disown it.
2214 */
2215 mtx_lock(&umtx_lock);
2216 if (pi->pi_owner == td)
2217 umtx_pi_disown(pi);
2218 mtx_unlock(&umtx_lock);
2219 }
2220 }
2221 return (0);
2222 }
2223
2224 /*
2225 * Lock a PI mutex.
2226 */
2227 static int
do_lock_pi(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2228 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
2229 struct _umtx_time *timeout, int try)
2230 {
2231 struct umtx_abs_timeout timo;
2232 struct umtx_q *uq;
2233 struct umtx_pi *pi, *new_pi;
2234 uint32_t id, old_owner, owner, old;
2235 int error, rv;
2236
2237 id = td->td_tid;
2238 uq = td->td_umtxq;
2239
2240 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2241 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2242 &uq->uq_key)) != 0)
2243 return (error);
2244
2245 if (timeout != NULL)
2246 umtx_abs_timeout_init2(&timo, timeout);
2247
2248 umtxq_lock(&uq->uq_key);
2249 pi = umtx_pi_lookup(&uq->uq_key);
2250 if (pi == NULL) {
2251 new_pi = umtx_pi_alloc(M_NOWAIT);
2252 if (new_pi == NULL) {
2253 umtxq_unlock(&uq->uq_key);
2254 new_pi = umtx_pi_alloc(M_WAITOK);
2255 umtxq_lock(&uq->uq_key);
2256 pi = umtx_pi_lookup(&uq->uq_key);
2257 if (pi != NULL) {
2258 umtx_pi_free(new_pi);
2259 new_pi = NULL;
2260 }
2261 }
2262 if (new_pi != NULL) {
2263 new_pi->pi_key = uq->uq_key;
2264 umtx_pi_insert(new_pi);
2265 pi = new_pi;
2266 }
2267 }
2268 umtx_pi_ref(pi);
2269 umtxq_unlock(&uq->uq_key);
2270
2271 /*
2272 * Care must be exercised when dealing with umtx structure. It
2273 * can fault on any access.
2274 */
2275 for (;;) {
2276 /*
2277 * Try the uncontested case. This should be done in userland.
2278 */
2279 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
2280 /* The address was invalid. */
2281 if (rv == -1) {
2282 error = EFAULT;
2283 break;
2284 }
2285 /* The acquire succeeded. */
2286 if (rv == 0) {
2287 MPASS(owner == UMUTEX_UNOWNED);
2288 error = 0;
2289 break;
2290 }
2291
2292 if (owner == UMUTEX_RB_NOTRECOV) {
2293 error = ENOTRECOVERABLE;
2294 break;
2295 }
2296
2297 /*
2298 * Nobody owns it, but the acquire failed. This can happen
2299 * with ll/sc atomics.
2300 */
2301 if (owner == UMUTEX_UNOWNED) {
2302 error = thread_check_susp(td, true);
2303 if (error != 0)
2304 break;
2305 continue;
2306 }
2307
2308 /*
2309 * Avoid overwriting a possible error from sleep due
2310 * to the pending signal with suspension check result.
2311 */
2312 if (error == 0) {
2313 error = thread_check_susp(td, true);
2314 if (error != 0)
2315 break;
2316 }
2317
2318 /* If no one owns it but it is contested try to acquire it. */
2319 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) {
2320 old_owner = owner;
2321 rv = casueword32(&m->m_owner, owner, &owner,
2322 id | UMUTEX_CONTESTED);
2323 /* The address was invalid. */
2324 if (rv == -1) {
2325 error = EFAULT;
2326 break;
2327 }
2328 if (rv == 1) {
2329 if (error == 0) {
2330 error = thread_check_susp(td, true);
2331 if (error != 0)
2332 break;
2333 }
2334
2335 /*
2336 * If this failed the lock could
2337 * changed, restart.
2338 */
2339 continue;
2340 }
2341
2342 MPASS(rv == 0);
2343 MPASS(owner == old_owner);
2344 umtxq_lock(&uq->uq_key);
2345 umtxq_busy(&uq->uq_key);
2346 error = umtx_pi_claim(pi, td);
2347 umtxq_unbusy(&uq->uq_key);
2348 umtxq_unlock(&uq->uq_key);
2349 if (error != 0) {
2350 /*
2351 * Since we're going to return an
2352 * error, restore the m_owner to its
2353 * previous, unowned state to avoid
2354 * compounding the problem.
2355 */
2356 (void)casuword32(&m->m_owner,
2357 id | UMUTEX_CONTESTED, old_owner);
2358 }
2359 if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
2360 error = EOWNERDEAD;
2361 break;
2362 }
2363
2364 if ((owner & ~UMUTEX_CONTESTED) == id) {
2365 error = EDEADLK;
2366 break;
2367 }
2368
2369 if (try != 0) {
2370 error = EBUSY;
2371 break;
2372 }
2373
2374 /*
2375 * If we caught a signal, we have retried and now
2376 * exit immediately.
2377 */
2378 if (error != 0)
2379 break;
2380
2381 umtxq_busy_unlocked(&uq->uq_key);
2382
2383 /*
2384 * Set the contested bit so that a release in user space
2385 * knows to use the system call for unlock. If this fails
2386 * either some one else has acquired the lock or it has been
2387 * released.
2388 */
2389 rv = casueword32(&m->m_owner, owner, &old, owner |
2390 UMUTEX_CONTESTED);
2391
2392 /* The address was invalid. */
2393 if (rv == -1) {
2394 umtxq_unbusy_unlocked(&uq->uq_key);
2395 error = EFAULT;
2396 break;
2397 }
2398 if (rv == 1) {
2399 umtxq_unbusy_unlocked(&uq->uq_key);
2400 error = thread_check_susp(td, true);
2401 if (error != 0)
2402 break;
2403
2404 /*
2405 * The lock changed and we need to retry or we
2406 * lost a race to the thread unlocking the
2407 * umtx. Note that the UMUTEX_RB_OWNERDEAD
2408 * value for owner is impossible there.
2409 */
2410 continue;
2411 }
2412
2413 umtxq_lock(&uq->uq_key);
2414
2415 /* We set the contested bit, sleep. */
2416 MPASS(old == owner);
2417 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
2418 "umtxpi", timeout == NULL ? NULL : &timo,
2419 (flags & USYNC_PROCESS_SHARED) != 0);
2420 if (error != 0)
2421 continue;
2422
2423 error = thread_check_susp(td, false);
2424 if (error != 0)
2425 break;
2426 }
2427
2428 umtxq_lock(&uq->uq_key);
2429 umtx_pi_unref(pi);
2430 umtxq_unlock(&uq->uq_key);
2431
2432 umtx_key_release(&uq->uq_key);
2433 return (error);
2434 }
2435
2436 /*
2437 * Unlock a PI mutex.
2438 */
2439 static int
do_unlock_pi(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2440 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2441 {
2442 struct umtx_key key;
2443 uint32_t id, new_owner, old, owner;
2444 int count, error;
2445
2446 id = td->td_tid;
2447
2448 usrloop:
2449 /*
2450 * Make sure we own this mtx.
2451 */
2452 error = fueword32(&m->m_owner, &owner);
2453 if (error == -1)
2454 return (EFAULT);
2455
2456 if ((owner & ~UMUTEX_CONTESTED) != id)
2457 return (EPERM);
2458
2459 new_owner = umtx_unlock_val(flags, rb);
2460
2461 /* This should be done in userland */
2462 if ((owner & UMUTEX_CONTESTED) == 0) {
2463 error = casueword32(&m->m_owner, owner, &old, new_owner);
2464 if (error == -1)
2465 return (EFAULT);
2466 if (error == 1) {
2467 error = thread_check_susp(td, true);
2468 if (error != 0)
2469 return (error);
2470 goto usrloop;
2471 }
2472 if (old == owner)
2473 return (0);
2474 owner = old;
2475 }
2476
2477 /* We should only ever be in here for contested locks */
2478 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2479 TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2480 &key)) != 0)
2481 return (error);
2482
2483 umtxq_lock(&key);
2484 umtxq_busy(&key);
2485 error = umtx_pi_drop(td, &key, rb, &count);
2486 if (error != 0) {
2487 umtxq_unbusy(&key);
2488 umtxq_unlock(&key);
2489 umtx_key_release(&key);
2490 /* userland messed the mutex */
2491 return (error);
2492 }
2493 umtxq_unlock(&key);
2494
2495 /*
2496 * When unlocking the umtx, it must be marked as unowned if
2497 * there is zero or one thread only waiting for it.
2498 * Otherwise, it must be marked as contested.
2499 */
2500
2501 if (count > 1)
2502 new_owner |= UMUTEX_CONTESTED;
2503 again:
2504 error = casueword32(&m->m_owner, owner, &old, new_owner);
2505 if (error == 1) {
2506 error = thread_check_susp(td, false);
2507 if (error == 0)
2508 goto again;
2509 }
2510 umtxq_unbusy_unlocked(&key);
2511 umtx_key_release(&key);
2512 if (error == -1)
2513 return (EFAULT);
2514 if (error == 0 && old != owner)
2515 return (EINVAL);
2516 return (error);
2517 }
2518
2519 /*
2520 * Lock a PP mutex.
2521 */
2522 static int
do_lock_pp(struct thread * td,struct umutex * m,uint32_t flags,struct _umtx_time * timeout,int try)2523 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
2524 struct _umtx_time *timeout, int try)
2525 {
2526 struct umtx_abs_timeout timo;
2527 struct umtx_q *uq, *uq2;
2528 struct umtx_pi *pi;
2529 uint32_t ceiling;
2530 uint32_t owner, id;
2531 int error, pri, old_inherited_pri, new_pri, rv;
2532 bool su;
2533
2534 id = td->td_tid;
2535 uq = td->td_umtxq;
2536 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2537 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2538 &uq->uq_key)) != 0)
2539 return (error);
2540
2541 if (timeout != NULL)
2542 umtx_abs_timeout_init2(&timo, timeout);
2543
2544 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2545 for (;;) {
2546 old_inherited_pri = uq->uq_inherited_pri;
2547 umtxq_busy_unlocked(&uq->uq_key);
2548
2549 rv = fueword32(&m->m_ceilings[0], &ceiling);
2550 if (rv == -1) {
2551 error = EFAULT;
2552 goto out;
2553 }
2554 ceiling = RTP_PRIO_MAX - ceiling;
2555 if (ceiling > RTP_PRIO_MAX) {
2556 error = EINVAL;
2557 goto out;
2558 }
2559 new_pri = PRI_MIN_REALTIME + ceiling;
2560
2561 if (td->td_base_user_pri < new_pri) {
2562 error = EINVAL;
2563 goto out;
2564 }
2565 if (su) {
2566 mtx_lock(&umtx_lock);
2567 if (new_pri < uq->uq_inherited_pri) {
2568 uq->uq_inherited_pri = new_pri;
2569 thread_lock(td);
2570 if (new_pri < UPRI(td))
2571 sched_lend_user_prio(td, new_pri);
2572 thread_unlock(td);
2573 }
2574 mtx_unlock(&umtx_lock);
2575 }
2576
2577 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2578 id | UMUTEX_CONTESTED);
2579 /* The address was invalid. */
2580 if (rv == -1) {
2581 error = EFAULT;
2582 break;
2583 }
2584 if (rv == 0) {
2585 MPASS(owner == UMUTEX_CONTESTED);
2586 error = 0;
2587 break;
2588 }
2589 /* rv == 1 */
2590 if (owner == UMUTEX_RB_OWNERDEAD) {
2591 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
2592 &owner, id | UMUTEX_CONTESTED);
2593 if (rv == -1) {
2594 error = EFAULT;
2595 break;
2596 }
2597 if (rv == 0) {
2598 MPASS(owner == UMUTEX_RB_OWNERDEAD);
2599 error = EOWNERDEAD; /* success */
2600 break;
2601 }
2602
2603 /*
2604 * rv == 1, only check for suspension if we
2605 * did not already catched a signal. If we
2606 * get an error from the check, the same
2607 * condition is checked by the umtxq_sleep()
2608 * call below, so we should obliterate the
2609 * error to not skip the last loop iteration.
2610 */
2611 if (error == 0) {
2612 error = thread_check_susp(td, false);
2613 if (error == 0 && try == 0) {
2614 umtxq_unbusy_unlocked(&uq->uq_key);
2615 continue;
2616 }
2617 error = 0;
2618 }
2619 } else if (owner == UMUTEX_RB_NOTRECOV) {
2620 error = ENOTRECOVERABLE;
2621 } else if (owner == UMUTEX_CONTESTED) {
2622 /* Spurious failure, retry. */
2623 umtxq_unbusy_unlocked(&uq->uq_key);
2624 continue;
2625 }
2626
2627 if (try != 0)
2628 error = EBUSY;
2629
2630 /*
2631 * If we caught a signal, we have retried and now
2632 * exit immediately.
2633 */
2634 if (error != 0)
2635 break;
2636
2637 umtxq_lock(&uq->uq_key);
2638 umtxq_insert(uq);
2639 umtxq_unbusy(&uq->uq_key);
2640 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2641 NULL : &timo);
2642 umtxq_remove(uq);
2643 umtxq_unlock(&uq->uq_key);
2644
2645 mtx_lock(&umtx_lock);
2646 uq->uq_inherited_pri = old_inherited_pri;
2647 pri = PRI_MAX;
2648 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2649 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2650 if (uq2 != NULL) {
2651 if (pri > UPRI(uq2->uq_thread))
2652 pri = UPRI(uq2->uq_thread);
2653 }
2654 }
2655 if (pri > uq->uq_inherited_pri)
2656 pri = uq->uq_inherited_pri;
2657 thread_lock(td);
2658 sched_lend_user_prio(td, pri);
2659 thread_unlock(td);
2660 mtx_unlock(&umtx_lock);
2661 }
2662
2663 if (error != 0 && error != EOWNERDEAD) {
2664 mtx_lock(&umtx_lock);
2665 uq->uq_inherited_pri = old_inherited_pri;
2666 pri = PRI_MAX;
2667 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2668 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2669 if (uq2 != NULL) {
2670 if (pri > UPRI(uq2->uq_thread))
2671 pri = UPRI(uq2->uq_thread);
2672 }
2673 }
2674 if (pri > uq->uq_inherited_pri)
2675 pri = uq->uq_inherited_pri;
2676 thread_lock(td);
2677 sched_lend_user_prio(td, pri);
2678 thread_unlock(td);
2679 mtx_unlock(&umtx_lock);
2680 }
2681
2682 out:
2683 umtxq_unbusy_unlocked(&uq->uq_key);
2684 umtx_key_release(&uq->uq_key);
2685 return (error);
2686 }
2687
2688 /*
2689 * Unlock a PP mutex.
2690 */
2691 static int
do_unlock_pp(struct thread * td,struct umutex * m,uint32_t flags,bool rb)2692 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2693 {
2694 struct umtx_key key;
2695 struct umtx_q *uq, *uq2;
2696 struct umtx_pi *pi;
2697 uint32_t id, owner, rceiling;
2698 int error, pri, new_inherited_pri;
2699 bool su;
2700
2701 id = td->td_tid;
2702 uq = td->td_umtxq;
2703 su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2704
2705 /*
2706 * Make sure we own this mtx.
2707 */
2708 error = fueword32(&m->m_owner, &owner);
2709 if (error == -1)
2710 return (EFAULT);
2711
2712 if ((owner & ~UMUTEX_CONTESTED) != id)
2713 return (EPERM);
2714
2715 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2716 if (error != 0)
2717 return (error);
2718
2719 if (rceiling == -1)
2720 new_inherited_pri = PRI_MAX;
2721 else {
2722 rceiling = RTP_PRIO_MAX - rceiling;
2723 if (rceiling > RTP_PRIO_MAX)
2724 return (EINVAL);
2725 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2726 }
2727
2728 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2729 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2730 &key)) != 0)
2731 return (error);
2732 umtxq_busy_unlocked(&key);
2733
2734 /*
2735 * For priority protected mutex, always set unlocked state
2736 * to UMUTEX_CONTESTED, so that userland always enters kernel
2737 * to lock the mutex, it is necessary because thread priority
2738 * has to be adjusted for such mutex.
2739 */
2740 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
2741 UMUTEX_CONTESTED);
2742
2743 umtxq_lock(&key);
2744 if (error == 0)
2745 umtxq_signal(&key, 1);
2746 umtxq_unbusy(&key);
2747 umtxq_unlock(&key);
2748
2749 if (error == -1)
2750 error = EFAULT;
2751 else {
2752 mtx_lock(&umtx_lock);
2753 if (su || new_inherited_pri == PRI_MAX)
2754 uq->uq_inherited_pri = new_inherited_pri;
2755 pri = PRI_MAX;
2756 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2757 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2758 if (uq2 != NULL) {
2759 if (pri > UPRI(uq2->uq_thread))
2760 pri = UPRI(uq2->uq_thread);
2761 }
2762 }
2763 if (pri > uq->uq_inherited_pri)
2764 pri = uq->uq_inherited_pri;
2765 thread_lock(td);
2766 sched_lend_user_prio(td, pri);
2767 thread_unlock(td);
2768 mtx_unlock(&umtx_lock);
2769 }
2770 umtx_key_release(&key);
2771 return (error);
2772 }
2773
2774 static int
do_set_ceiling(struct thread * td,struct umutex * m,uint32_t ceiling,uint32_t * old_ceiling)2775 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2776 uint32_t *old_ceiling)
2777 {
2778 struct umtx_q *uq;
2779 uint32_t flags, id, owner, save_ceiling;
2780 int error, rv, rv1;
2781
2782 error = fueword32(&m->m_flags, &flags);
2783 if (error == -1)
2784 return (EFAULT);
2785 if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2786 return (EINVAL);
2787 if (ceiling > RTP_PRIO_MAX)
2788 return (EINVAL);
2789 id = td->td_tid;
2790 uq = td->td_umtxq;
2791 if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2792 TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2793 &uq->uq_key)) != 0)
2794 return (error);
2795 for (;;) {
2796 umtxq_busy_unlocked(&uq->uq_key);
2797
2798 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
2799 if (rv == -1) {
2800 error = EFAULT;
2801 break;
2802 }
2803
2804 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2805 id | UMUTEX_CONTESTED);
2806 if (rv == -1) {
2807 error = EFAULT;
2808 break;
2809 }
2810
2811 if (rv == 0) {
2812 MPASS(owner == UMUTEX_CONTESTED);
2813 rv = suword32(&m->m_ceilings[0], ceiling);
2814 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
2815 error = (rv == 0 && rv1 == 0) ? 0: EFAULT;
2816 break;
2817 }
2818
2819 if ((owner & ~UMUTEX_CONTESTED) == id) {
2820 rv = suword32(&m->m_ceilings[0], ceiling);
2821 error = rv == 0 ? 0 : EFAULT;
2822 break;
2823 }
2824
2825 if (owner == UMUTEX_RB_OWNERDEAD) {
2826 error = EOWNERDEAD;
2827 break;
2828 } else if (owner == UMUTEX_RB_NOTRECOV) {
2829 error = ENOTRECOVERABLE;
2830 break;
2831 } else if (owner == UMUTEX_CONTESTED) {
2832 /* Spurious failure, retry. */
2833 umtxq_unbusy_unlocked(&uq->uq_key);
2834 continue;
2835 }
2836
2837 /*
2838 * If we caught a signal, we have retried and now
2839 * exit immediately.
2840 */
2841 if (error != 0)
2842 break;
2843
2844 /*
2845 * We set the contested bit, sleep. Otherwise the lock changed
2846 * and we need to retry or we lost a race to the thread
2847 * unlocking the umtx.
2848 */
2849 umtxq_lock(&uq->uq_key);
2850 umtxq_insert(uq);
2851 umtxq_unbusy(&uq->uq_key);
2852 error = umtxq_sleep(uq, "umtxpp", NULL);
2853 umtxq_remove(uq);
2854 umtxq_unlock(&uq->uq_key);
2855 }
2856 umtxq_lock(&uq->uq_key);
2857 if (error == 0)
2858 umtxq_signal(&uq->uq_key, INT_MAX);
2859 umtxq_unbusy(&uq->uq_key);
2860 umtxq_unlock(&uq->uq_key);
2861 umtx_key_release(&uq->uq_key);
2862 if (error == 0 && old_ceiling != NULL) {
2863 rv = suword32(old_ceiling, save_ceiling);
2864 error = rv == 0 ? 0 : EFAULT;
2865 }
2866 return (error);
2867 }
2868
2869 /*
2870 * Lock a userland POSIX mutex.
2871 */
2872 static int
do_lock_umutex(struct thread * td,struct umutex * m,struct _umtx_time * timeout,int mode)2873 do_lock_umutex(struct thread *td, struct umutex *m,
2874 struct _umtx_time *timeout, int mode)
2875 {
2876 uint32_t flags;
2877 int error;
2878
2879 error = fueword32(&m->m_flags, &flags);
2880 if (error == -1)
2881 return (EFAULT);
2882
2883 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2884 case 0:
2885 error = do_lock_normal(td, m, flags, timeout, mode);
2886 break;
2887 case UMUTEX_PRIO_INHERIT:
2888 error = do_lock_pi(td, m, flags, timeout, mode);
2889 break;
2890 case UMUTEX_PRIO_PROTECT:
2891 error = do_lock_pp(td, m, flags, timeout, mode);
2892 break;
2893 default:
2894 return (EINVAL);
2895 }
2896 if (timeout == NULL) {
2897 if (error == EINTR && mode != _UMUTEX_WAIT)
2898 error = ERESTART;
2899 } else {
2900 /* Timed-locking is not restarted. */
2901 if (error == ERESTART)
2902 error = EINTR;
2903 }
2904 return (error);
2905 }
2906
2907 /*
2908 * Unlock a userland POSIX mutex.
2909 */
2910 static int
do_unlock_umutex(struct thread * td,struct umutex * m,bool rb)2911 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb)
2912 {
2913 uint32_t flags;
2914 int error;
2915
2916 error = fueword32(&m->m_flags, &flags);
2917 if (error == -1)
2918 return (EFAULT);
2919
2920 switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2921 case 0:
2922 return (do_unlock_normal(td, m, flags, rb));
2923 case UMUTEX_PRIO_INHERIT:
2924 return (do_unlock_pi(td, m, flags, rb));
2925 case UMUTEX_PRIO_PROTECT:
2926 return (do_unlock_pp(td, m, flags, rb));
2927 }
2928
2929 return (EINVAL);
2930 }
2931
2932 static int
do_cv_wait(struct thread * td,struct ucond * cv,struct umutex * m,struct timespec * timeout,u_long wflags)2933 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2934 struct timespec *timeout, u_long wflags)
2935 {
2936 struct umtx_abs_timeout timo;
2937 struct umtx_q *uq;
2938 uint32_t flags, clockid, hasw;
2939 int error;
2940
2941 uq = td->td_umtxq;
2942 error = fueword32(&cv->c_flags, &flags);
2943 if (error == -1)
2944 return (EFAULT);
2945 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2946 if (error != 0)
2947 return (error);
2948
2949 if ((wflags & CVWAIT_CLOCKID) != 0) {
2950 error = fueword32(&cv->c_clockid, &clockid);
2951 if (error == -1) {
2952 umtx_key_release(&uq->uq_key);
2953 return (EFAULT);
2954 }
2955 if (clockid < CLOCK_REALTIME ||
2956 clockid >= CLOCK_THREAD_CPUTIME_ID) {
2957 /* hmm, only HW clock id will work. */
2958 umtx_key_release(&uq->uq_key);
2959 return (EINVAL);
2960 }
2961 } else {
2962 clockid = CLOCK_REALTIME;
2963 }
2964
2965 umtxq_lock(&uq->uq_key);
2966 umtxq_busy(&uq->uq_key);
2967 umtxq_insert(uq);
2968 umtxq_unlock(&uq->uq_key);
2969
2970 /*
2971 * Set c_has_waiters to 1 before releasing user mutex, also
2972 * don't modify cache line when unnecessary.
2973 */
2974 error = fueword32(&cv->c_has_waiters, &hasw);
2975 if (error == 0 && hasw == 0)
2976 error = suword32(&cv->c_has_waiters, 1);
2977 if (error != 0) {
2978 umtxq_lock(&uq->uq_key);
2979 umtxq_remove(uq);
2980 umtxq_unbusy(&uq->uq_key);
2981 error = EFAULT;
2982 goto out;
2983 }
2984
2985 umtxq_unbusy_unlocked(&uq->uq_key);
2986
2987 error = do_unlock_umutex(td, m, false);
2988
2989 if (timeout != NULL)
2990 umtx_abs_timeout_init(&timo, clockid,
2991 (wflags & CVWAIT_ABSTIME) != 0, timeout);
2992
2993 umtxq_lock(&uq->uq_key);
2994 if (error == 0) {
2995 error = umtxq_sleep(uq, "ucond", timeout == NULL ?
2996 NULL : &timo);
2997 }
2998
2999 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3000 error = 0;
3001 else {
3002 /*
3003 * This must be timeout,interrupted by signal or
3004 * surprious wakeup, clear c_has_waiter flag when
3005 * necessary.
3006 */
3007 umtxq_busy(&uq->uq_key);
3008 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
3009 int oldlen = uq->uq_cur_queue->length;
3010 umtxq_remove(uq);
3011 if (oldlen == 1) {
3012 umtxq_unlock(&uq->uq_key);
3013 if (suword32(&cv->c_has_waiters, 0) != 0 &&
3014 error == 0)
3015 error = EFAULT;
3016 umtxq_lock(&uq->uq_key);
3017 }
3018 }
3019 umtxq_unbusy(&uq->uq_key);
3020 if (error == ERESTART)
3021 error = EINTR;
3022 }
3023 out:
3024 umtxq_unlock(&uq->uq_key);
3025 umtx_key_release(&uq->uq_key);
3026 return (error);
3027 }
3028
3029 /*
3030 * Signal a userland condition variable.
3031 */
3032 static int
do_cv_signal(struct thread * td,struct ucond * cv)3033 do_cv_signal(struct thread *td, struct ucond *cv)
3034 {
3035 struct umtx_key key;
3036 int error, cnt, nwake;
3037 uint32_t flags;
3038
3039 error = fueword32(&cv->c_flags, &flags);
3040 if (error == -1)
3041 return (EFAULT);
3042 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3043 return (error);
3044 umtxq_lock(&key);
3045 umtxq_busy(&key);
3046 cnt = umtxq_count(&key);
3047 nwake = umtxq_signal(&key, 1);
3048 if (cnt <= nwake) {
3049 umtxq_unlock(&key);
3050 error = suword32(&cv->c_has_waiters, 0);
3051 if (error == -1)
3052 error = EFAULT;
3053 umtxq_lock(&key);
3054 }
3055 umtxq_unbusy(&key);
3056 umtxq_unlock(&key);
3057 umtx_key_release(&key);
3058 return (error);
3059 }
3060
3061 static int
do_cv_broadcast(struct thread * td,struct ucond * cv)3062 do_cv_broadcast(struct thread *td, struct ucond *cv)
3063 {
3064 struct umtx_key key;
3065 int error;
3066 uint32_t flags;
3067
3068 error = fueword32(&cv->c_flags, &flags);
3069 if (error == -1)
3070 return (EFAULT);
3071 if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
3072 return (error);
3073
3074 umtxq_lock(&key);
3075 umtxq_busy(&key);
3076 umtxq_signal(&key, INT_MAX);
3077 umtxq_unlock(&key);
3078
3079 error = suword32(&cv->c_has_waiters, 0);
3080 if (error == -1)
3081 error = EFAULT;
3082
3083 umtxq_unbusy_unlocked(&key);
3084
3085 umtx_key_release(&key);
3086 return (error);
3087 }
3088
3089 static int
do_rw_rdlock(struct thread * td,struct urwlock * rwlock,long fflag,struct _umtx_time * timeout)3090 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
3091 struct _umtx_time *timeout)
3092 {
3093 struct umtx_abs_timeout timo;
3094 struct umtx_q *uq;
3095 uint32_t flags, wrflags;
3096 int32_t state, oldstate;
3097 int32_t blocked_readers;
3098 int error, error1, rv;
3099
3100 uq = td->td_umtxq;
3101 error = fueword32(&rwlock->rw_flags, &flags);
3102 if (error == -1)
3103 return (EFAULT);
3104 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3105 if (error != 0)
3106 return (error);
3107
3108 if (timeout != NULL)
3109 umtx_abs_timeout_init2(&timo, timeout);
3110
3111 wrflags = URWLOCK_WRITE_OWNER;
3112 if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
3113 wrflags |= URWLOCK_WRITE_WAITERS;
3114
3115 for (;;) {
3116 rv = fueword32(&rwlock->rw_state, &state);
3117 if (rv == -1) {
3118 umtx_key_release(&uq->uq_key);
3119 return (EFAULT);
3120 }
3121
3122 /* try to lock it */
3123 while (!(state & wrflags)) {
3124 if (__predict_false(URWLOCK_READER_COUNT(state) ==
3125 URWLOCK_MAX_READERS)) {
3126 umtx_key_release(&uq->uq_key);
3127 return (EAGAIN);
3128 }
3129 rv = casueword32(&rwlock->rw_state, state,
3130 &oldstate, state + 1);
3131 if (rv == -1) {
3132 umtx_key_release(&uq->uq_key);
3133 return (EFAULT);
3134 }
3135 if (rv == 0) {
3136 MPASS(oldstate == state);
3137 umtx_key_release(&uq->uq_key);
3138 return (0);
3139 }
3140 error = thread_check_susp(td, true);
3141 if (error != 0)
3142 break;
3143 state = oldstate;
3144 }
3145
3146 if (error)
3147 break;
3148
3149 /* grab monitor lock */
3150 umtxq_busy_unlocked(&uq->uq_key);
3151
3152 /*
3153 * re-read the state, in case it changed between the try-lock above
3154 * and the check below
3155 */
3156 rv = fueword32(&rwlock->rw_state, &state);
3157 if (rv == -1)
3158 error = EFAULT;
3159
3160 /* set read contention bit */
3161 while (error == 0 && (state & wrflags) &&
3162 !(state & URWLOCK_READ_WAITERS)) {
3163 rv = casueword32(&rwlock->rw_state, state,
3164 &oldstate, state | URWLOCK_READ_WAITERS);
3165 if (rv == -1) {
3166 error = EFAULT;
3167 break;
3168 }
3169 if (rv == 0) {
3170 MPASS(oldstate == state);
3171 goto sleep;
3172 }
3173 state = oldstate;
3174 error = thread_check_susp(td, false);
3175 if (error != 0)
3176 break;
3177 }
3178 if (error != 0) {
3179 umtxq_unbusy_unlocked(&uq->uq_key);
3180 break;
3181 }
3182
3183 /* state is changed while setting flags, restart */
3184 if (!(state & wrflags)) {
3185 umtxq_unbusy_unlocked(&uq->uq_key);
3186 error = thread_check_susp(td, true);
3187 if (error != 0)
3188 break;
3189 continue;
3190 }
3191
3192 sleep:
3193 /*
3194 * Contention bit is set, before sleeping, increase
3195 * read waiter count.
3196 */
3197 rv = fueword32(&rwlock->rw_blocked_readers,
3198 &blocked_readers);
3199 if (rv == 0)
3200 rv = suword32(&rwlock->rw_blocked_readers,
3201 blocked_readers + 1);
3202 if (rv == -1) {
3203 umtxq_unbusy_unlocked(&uq->uq_key);
3204 error = EFAULT;
3205 break;
3206 }
3207
3208 while (state & wrflags) {
3209 umtxq_lock(&uq->uq_key);
3210 umtxq_insert(uq);
3211 umtxq_unbusy(&uq->uq_key);
3212
3213 error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
3214 NULL : &timo);
3215
3216 umtxq_busy(&uq->uq_key);
3217 umtxq_remove(uq);
3218 umtxq_unlock(&uq->uq_key);
3219 if (error)
3220 break;
3221 rv = fueword32(&rwlock->rw_state, &state);
3222 if (rv == -1) {
3223 error = EFAULT;
3224 break;
3225 }
3226 }
3227
3228 /* decrease read waiter count, and may clear read contention bit */
3229 rv = fueword32(&rwlock->rw_blocked_readers,
3230 &blocked_readers);
3231 if (rv == 0)
3232 rv = suword32(&rwlock->rw_blocked_readers,
3233 blocked_readers - 1);
3234 if (rv == -1) {
3235 umtxq_unbusy_unlocked(&uq->uq_key);
3236 error = EFAULT;
3237 break;
3238 }
3239 if (blocked_readers == 1) {
3240 rv = fueword32(&rwlock->rw_state, &state);
3241 if (rv == -1) {
3242 umtxq_unbusy_unlocked(&uq->uq_key);
3243 error = EFAULT;
3244 break;
3245 }
3246 for (;;) {
3247 rv = casueword32(&rwlock->rw_state, state,
3248 &oldstate, state & ~URWLOCK_READ_WAITERS);
3249 if (rv == -1) {
3250 error = EFAULT;
3251 break;
3252 }
3253 if (rv == 0) {
3254 MPASS(oldstate == state);
3255 break;
3256 }
3257 state = oldstate;
3258 error1 = thread_check_susp(td, false);
3259 if (error1 != 0) {
3260 if (error == 0)
3261 error = error1;
3262 break;
3263 }
3264 }
3265 }
3266
3267 umtxq_unbusy_unlocked(&uq->uq_key);
3268 if (error != 0)
3269 break;
3270 }
3271 umtx_key_release(&uq->uq_key);
3272 if (error == ERESTART)
3273 error = EINTR;
3274 return (error);
3275 }
3276
3277 static int
do_rw_wrlock(struct thread * td,struct urwlock * rwlock,struct _umtx_time * timeout)3278 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout)
3279 {
3280 struct umtx_abs_timeout timo;
3281 struct umtx_q *uq;
3282 uint32_t flags;
3283 int32_t state, oldstate;
3284 int32_t blocked_writers;
3285 int32_t blocked_readers;
3286 int error, error1, rv;
3287
3288 uq = td->td_umtxq;
3289 error = fueword32(&rwlock->rw_flags, &flags);
3290 if (error == -1)
3291 return (EFAULT);
3292 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3293 if (error != 0)
3294 return (error);
3295
3296 if (timeout != NULL)
3297 umtx_abs_timeout_init2(&timo, timeout);
3298
3299 blocked_readers = 0;
3300 for (;;) {
3301 rv = fueword32(&rwlock->rw_state, &state);
3302 if (rv == -1) {
3303 umtx_key_release(&uq->uq_key);
3304 return (EFAULT);
3305 }
3306 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
3307 URWLOCK_READER_COUNT(state) == 0) {
3308 rv = casueword32(&rwlock->rw_state, state,
3309 &oldstate, state | URWLOCK_WRITE_OWNER);
3310 if (rv == -1) {
3311 umtx_key_release(&uq->uq_key);
3312 return (EFAULT);
3313 }
3314 if (rv == 0) {
3315 MPASS(oldstate == state);
3316 umtx_key_release(&uq->uq_key);
3317 return (0);
3318 }
3319 state = oldstate;
3320 error = thread_check_susp(td, true);
3321 if (error != 0)
3322 break;
3323 }
3324
3325 if (error) {
3326 if ((state & (URWLOCK_WRITE_OWNER |
3327 URWLOCK_WRITE_WAITERS)) == 0 &&
3328 blocked_readers != 0) {
3329 umtxq_lock(&uq->uq_key);
3330 umtxq_busy(&uq->uq_key);
3331 umtxq_signal_queue(&uq->uq_key, INT_MAX,
3332 UMTX_SHARED_QUEUE);
3333 umtxq_unbusy(&uq->uq_key);
3334 umtxq_unlock(&uq->uq_key);
3335 }
3336
3337 break;
3338 }
3339
3340 /* grab monitor lock */
3341 umtxq_busy_unlocked(&uq->uq_key);
3342
3343 /*
3344 * Re-read the state, in case it changed between the
3345 * try-lock above and the check below.
3346 */
3347 rv = fueword32(&rwlock->rw_state, &state);
3348 if (rv == -1)
3349 error = EFAULT;
3350
3351 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
3352 URWLOCK_READER_COUNT(state) != 0) &&
3353 (state & URWLOCK_WRITE_WAITERS) == 0) {
3354 rv = casueword32(&rwlock->rw_state, state,
3355 &oldstate, state | URWLOCK_WRITE_WAITERS);
3356 if (rv == -1) {
3357 error = EFAULT;
3358 break;
3359 }
3360 if (rv == 0) {
3361 MPASS(oldstate == state);
3362 goto sleep;
3363 }
3364 state = oldstate;
3365 error = thread_check_susp(td, false);
3366 if (error != 0)
3367 break;
3368 }
3369 if (error != 0) {
3370 umtxq_unbusy_unlocked(&uq->uq_key);
3371 break;
3372 }
3373
3374 if ((state & URWLOCK_WRITE_OWNER) == 0 &&
3375 URWLOCK_READER_COUNT(state) == 0) {
3376 umtxq_unbusy_unlocked(&uq->uq_key);
3377 error = thread_check_susp(td, false);
3378 if (error != 0)
3379 break;
3380 continue;
3381 }
3382 sleep:
3383 rv = fueword32(&rwlock->rw_blocked_writers,
3384 &blocked_writers);
3385 if (rv == 0)
3386 rv = suword32(&rwlock->rw_blocked_writers,
3387 blocked_writers + 1);
3388 if (rv == -1) {
3389 umtxq_unbusy_unlocked(&uq->uq_key);
3390 error = EFAULT;
3391 break;
3392 }
3393
3394 while ((state & URWLOCK_WRITE_OWNER) ||
3395 URWLOCK_READER_COUNT(state) != 0) {
3396 umtxq_lock(&uq->uq_key);
3397 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3398 umtxq_unbusy(&uq->uq_key);
3399
3400 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
3401 NULL : &timo);
3402
3403 umtxq_busy(&uq->uq_key);
3404 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3405 umtxq_unlock(&uq->uq_key);
3406 if (error)
3407 break;
3408 rv = fueword32(&rwlock->rw_state, &state);
3409 if (rv == -1) {
3410 error = EFAULT;
3411 break;
3412 }
3413 }
3414
3415 rv = fueword32(&rwlock->rw_blocked_writers,
3416 &blocked_writers);
3417 if (rv == 0)
3418 rv = suword32(&rwlock->rw_blocked_writers,
3419 blocked_writers - 1);
3420 if (rv == -1) {
3421 umtxq_unbusy_unlocked(&uq->uq_key);
3422 error = EFAULT;
3423 break;
3424 }
3425 if (blocked_writers == 1) {
3426 rv = fueword32(&rwlock->rw_state, &state);
3427 if (rv == -1) {
3428 umtxq_unbusy_unlocked(&uq->uq_key);
3429 error = EFAULT;
3430 break;
3431 }
3432 for (;;) {
3433 rv = casueword32(&rwlock->rw_state, state,
3434 &oldstate, state & ~URWLOCK_WRITE_WAITERS);
3435 if (rv == -1) {
3436 error = EFAULT;
3437 break;
3438 }
3439 if (rv == 0) {
3440 MPASS(oldstate == state);
3441 break;
3442 }
3443 state = oldstate;
3444 error1 = thread_check_susp(td, false);
3445 /*
3446 * We are leaving the URWLOCK_WRITE_WAITERS
3447 * behind, but this should not harm the
3448 * correctness.
3449 */
3450 if (error1 != 0) {
3451 if (error == 0)
3452 error = error1;
3453 break;
3454 }
3455 }
3456 rv = fueword32(&rwlock->rw_blocked_readers,
3457 &blocked_readers);
3458 if (rv == -1) {
3459 umtxq_unbusy_unlocked(&uq->uq_key);
3460 error = EFAULT;
3461 break;
3462 }
3463 } else
3464 blocked_readers = 0;
3465
3466 umtxq_unbusy_unlocked(&uq->uq_key);
3467 }
3468
3469 umtx_key_release(&uq->uq_key);
3470 if (error == ERESTART)
3471 error = EINTR;
3472 return (error);
3473 }
3474
3475 static int
do_rw_unlock(struct thread * td,struct urwlock * rwlock)3476 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
3477 {
3478 struct umtx_q *uq;
3479 uint32_t flags;
3480 int32_t state, oldstate;
3481 int error, rv, q, count;
3482
3483 uq = td->td_umtxq;
3484 error = fueword32(&rwlock->rw_flags, &flags);
3485 if (error == -1)
3486 return (EFAULT);
3487 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3488 if (error != 0)
3489 return (error);
3490
3491 error = fueword32(&rwlock->rw_state, &state);
3492 if (error == -1) {
3493 error = EFAULT;
3494 goto out;
3495 }
3496 if (state & URWLOCK_WRITE_OWNER) {
3497 for (;;) {
3498 rv = casueword32(&rwlock->rw_state, state,
3499 &oldstate, state & ~URWLOCK_WRITE_OWNER);
3500 if (rv == -1) {
3501 error = EFAULT;
3502 goto out;
3503 }
3504 if (rv == 1) {
3505 state = oldstate;
3506 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
3507 error = EPERM;
3508 goto out;
3509 }
3510 error = thread_check_susp(td, true);
3511 if (error != 0)
3512 goto out;
3513 } else
3514 break;
3515 }
3516 } else if (URWLOCK_READER_COUNT(state) != 0) {
3517 for (;;) {
3518 rv = casueword32(&rwlock->rw_state, state,
3519 &oldstate, state - 1);
3520 if (rv == -1) {
3521 error = EFAULT;
3522 goto out;
3523 }
3524 if (rv == 1) {
3525 state = oldstate;
3526 if (URWLOCK_READER_COUNT(oldstate) == 0) {
3527 error = EPERM;
3528 goto out;
3529 }
3530 error = thread_check_susp(td, true);
3531 if (error != 0)
3532 goto out;
3533 } else
3534 break;
3535 }
3536 } else {
3537 error = EPERM;
3538 goto out;
3539 }
3540
3541 count = 0;
3542
3543 if (!(flags & URWLOCK_PREFER_READER)) {
3544 if (state & URWLOCK_WRITE_WAITERS) {
3545 count = 1;
3546 q = UMTX_EXCLUSIVE_QUEUE;
3547 } else if (state & URWLOCK_READ_WAITERS) {
3548 count = INT_MAX;
3549 q = UMTX_SHARED_QUEUE;
3550 }
3551 } else {
3552 if (state & URWLOCK_READ_WAITERS) {
3553 count = INT_MAX;
3554 q = UMTX_SHARED_QUEUE;
3555 } else if (state & URWLOCK_WRITE_WAITERS) {
3556 count = 1;
3557 q = UMTX_EXCLUSIVE_QUEUE;
3558 }
3559 }
3560
3561 if (count) {
3562 umtxq_lock(&uq->uq_key);
3563 umtxq_busy(&uq->uq_key);
3564 umtxq_signal_queue(&uq->uq_key, count, q);
3565 umtxq_unbusy(&uq->uq_key);
3566 umtxq_unlock(&uq->uq_key);
3567 }
3568 out:
3569 umtx_key_release(&uq->uq_key);
3570 return (error);
3571 }
3572
3573 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
3574 static int
do_sem_wait(struct thread * td,struct _usem * sem,struct _umtx_time * timeout)3575 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
3576 {
3577 struct umtx_abs_timeout timo;
3578 struct umtx_q *uq;
3579 uint32_t flags, count, count1;
3580 int error, rv, rv1;
3581
3582 uq = td->td_umtxq;
3583 error = fueword32(&sem->_flags, &flags);
3584 if (error == -1)
3585 return (EFAULT);
3586 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3587 if (error != 0)
3588 return (error);
3589
3590 if (timeout != NULL)
3591 umtx_abs_timeout_init2(&timo, timeout);
3592
3593 again:
3594 umtxq_lock(&uq->uq_key);
3595 umtxq_busy(&uq->uq_key);
3596 umtxq_insert(uq);
3597 umtxq_unlock(&uq->uq_key);
3598 rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
3599 if (rv != -1)
3600 rv1 = fueword32(&sem->_count, &count);
3601 if (rv == -1 || rv1 == -1 || count != 0 || (rv == 1 && count1 == 0)) {
3602 if (rv == 0)
3603 rv = suword32(&sem->_has_waiters, 0);
3604 umtxq_lock(&uq->uq_key);
3605 umtxq_unbusy(&uq->uq_key);
3606 umtxq_remove(uq);
3607 umtxq_unlock(&uq->uq_key);
3608 if (rv == -1 || rv1 == -1) {
3609 error = EFAULT;
3610 goto out;
3611 }
3612 if (count != 0) {
3613 error = 0;
3614 goto out;
3615 }
3616 MPASS(rv == 1 && count1 == 0);
3617 rv = thread_check_susp(td, true);
3618 if (rv == 0)
3619 goto again;
3620 error = rv;
3621 goto out;
3622 }
3623 umtxq_lock(&uq->uq_key);
3624 umtxq_unbusy(&uq->uq_key);
3625
3626 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3627
3628 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3629 error = 0;
3630 else {
3631 umtxq_remove(uq);
3632 /* A relative timeout cannot be restarted. */
3633 if (error == ERESTART && timeout != NULL &&
3634 (timeout->_flags & UMTX_ABSTIME) == 0)
3635 error = EINTR;
3636 }
3637 umtxq_unlock(&uq->uq_key);
3638 out:
3639 umtx_key_release(&uq->uq_key);
3640 return (error);
3641 }
3642
3643 /*
3644 * Signal a userland semaphore.
3645 */
3646 static int
do_sem_wake(struct thread * td,struct _usem * sem)3647 do_sem_wake(struct thread *td, struct _usem *sem)
3648 {
3649 struct umtx_key key;
3650 int error, cnt;
3651 uint32_t flags;
3652
3653 error = fueword32(&sem->_flags, &flags);
3654 if (error == -1)
3655 return (EFAULT);
3656 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3657 return (error);
3658 umtxq_lock(&key);
3659 umtxq_busy(&key);
3660 cnt = umtxq_count(&key);
3661 if (cnt > 0) {
3662 /*
3663 * Check if count is greater than 0, this means the memory is
3664 * still being referenced by user code, so we can safely
3665 * update _has_waiters flag.
3666 */
3667 if (cnt == 1) {
3668 umtxq_unlock(&key);
3669 error = suword32(&sem->_has_waiters, 0);
3670 umtxq_lock(&key);
3671 if (error == -1)
3672 error = EFAULT;
3673 }
3674 umtxq_signal(&key, 1);
3675 }
3676 umtxq_unbusy(&key);
3677 umtxq_unlock(&key);
3678 umtx_key_release(&key);
3679 return (error);
3680 }
3681 #endif
3682
3683 static int
do_sem2_wait(struct thread * td,struct _usem2 * sem,struct _umtx_time * timeout)3684 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
3685 {
3686 struct umtx_abs_timeout timo;
3687 struct umtx_q *uq;
3688 uint32_t count, flags;
3689 int error, rv;
3690
3691 uq = td->td_umtxq;
3692 flags = fuword32(&sem->_flags);
3693 if (timeout != NULL)
3694 umtx_abs_timeout_init2(&timo, timeout);
3695
3696 again:
3697 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3698 if (error != 0)
3699 return (error);
3700 umtxq_lock(&uq->uq_key);
3701 umtxq_busy(&uq->uq_key);
3702 umtxq_insert(uq);
3703 umtxq_unlock(&uq->uq_key);
3704 rv = fueword32(&sem->_count, &count);
3705 if (rv == -1) {
3706 umtxq_lock(&uq->uq_key);
3707 umtxq_unbusy(&uq->uq_key);
3708 umtxq_remove(uq);
3709 umtxq_unlock(&uq->uq_key);
3710 umtx_key_release(&uq->uq_key);
3711 return (EFAULT);
3712 }
3713 for (;;) {
3714 if (USEM_COUNT(count) != 0) {
3715 umtxq_lock(&uq->uq_key);
3716 umtxq_unbusy(&uq->uq_key);
3717 umtxq_remove(uq);
3718 umtxq_unlock(&uq->uq_key);
3719 umtx_key_release(&uq->uq_key);
3720 return (0);
3721 }
3722 if (count == USEM_HAS_WAITERS)
3723 break;
3724 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
3725 if (rv == 0)
3726 break;
3727 umtxq_lock(&uq->uq_key);
3728 umtxq_unbusy(&uq->uq_key);
3729 umtxq_remove(uq);
3730 umtxq_unlock(&uq->uq_key);
3731 umtx_key_release(&uq->uq_key);
3732 if (rv == -1)
3733 return (EFAULT);
3734 rv = thread_check_susp(td, true);
3735 if (rv != 0)
3736 return (rv);
3737 goto again;
3738 }
3739 umtxq_lock(&uq->uq_key);
3740 umtxq_unbusy(&uq->uq_key);
3741
3742 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3743
3744 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3745 error = 0;
3746 else {
3747 umtxq_remove(uq);
3748 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
3749 /* A relative timeout cannot be restarted. */
3750 if (error == ERESTART)
3751 error = EINTR;
3752 if (error == EINTR) {
3753 kern_clock_gettime(curthread, timo.clockid,
3754 &timo.cur);
3755 timespecsub(&timo.end, &timo.cur,
3756 &timeout->_timeout);
3757 }
3758 }
3759 }
3760 umtxq_unlock(&uq->uq_key);
3761 umtx_key_release(&uq->uq_key);
3762 return (error);
3763 }
3764
3765 /*
3766 * Signal a userland semaphore.
3767 */
3768 static int
do_sem2_wake(struct thread * td,struct _usem2 * sem)3769 do_sem2_wake(struct thread *td, struct _usem2 *sem)
3770 {
3771 struct umtx_key key;
3772 int error, cnt, rv;
3773 uint32_t count, flags;
3774
3775 rv = fueword32(&sem->_flags, &flags);
3776 if (rv == -1)
3777 return (EFAULT);
3778 if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3779 return (error);
3780 umtxq_lock(&key);
3781 umtxq_busy(&key);
3782 cnt = umtxq_count(&key);
3783 if (cnt > 0) {
3784 /*
3785 * If this was the last sleeping thread, clear the waiters
3786 * flag in _count.
3787 */
3788 if (cnt == 1) {
3789 umtxq_unlock(&key);
3790 rv = fueword32(&sem->_count, &count);
3791 while (rv != -1 && count & USEM_HAS_WAITERS) {
3792 rv = casueword32(&sem->_count, count, &count,
3793 count & ~USEM_HAS_WAITERS);
3794 if (rv == 1) {
3795 rv = thread_check_susp(td, true);
3796 if (rv != 0)
3797 break;
3798 }
3799 }
3800 if (rv == -1)
3801 error = EFAULT;
3802 else if (rv > 0) {
3803 error = rv;
3804 }
3805 umtxq_lock(&key);
3806 }
3807
3808 umtxq_signal(&key, 1);
3809 }
3810 umtxq_unbusy(&key);
3811 umtxq_unlock(&key);
3812 umtx_key_release(&key);
3813 return (error);
3814 }
3815
3816 #ifdef COMPAT_FREEBSD10
3817 int
freebsd10__umtx_lock(struct thread * td,struct freebsd10__umtx_lock_args * uap)3818 freebsd10__umtx_lock(struct thread *td, struct freebsd10__umtx_lock_args *uap)
3819 {
3820 return (do_lock_umtx(td, uap->umtx, td->td_tid, 0));
3821 }
3822
3823 int
freebsd10__umtx_unlock(struct thread * td,struct freebsd10__umtx_unlock_args * uap)3824 freebsd10__umtx_unlock(struct thread *td,
3825 struct freebsd10__umtx_unlock_args *uap)
3826 {
3827 return (do_unlock_umtx(td, uap->umtx, td->td_tid));
3828 }
3829 #endif
3830
3831 inline int
umtx_copyin_timeout(const void * uaddr,struct timespec * tsp)3832 umtx_copyin_timeout(const void *uaddr, struct timespec *tsp)
3833 {
3834 int error;
3835
3836 error = copyin(uaddr, tsp, sizeof(*tsp));
3837 if (error == 0) {
3838 if (!timespecvalid_interval(tsp))
3839 error = EINVAL;
3840 }
3841 return (error);
3842 }
3843
3844 static inline int
umtx_copyin_umtx_time(const void * uaddr,size_t size,struct _umtx_time * tp)3845 umtx_copyin_umtx_time(const void *uaddr, size_t size, struct _umtx_time *tp)
3846 {
3847 int error;
3848
3849 if (size <= sizeof(tp->_timeout)) {
3850 tp->_clockid = CLOCK_REALTIME;
3851 tp->_flags = 0;
3852 error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout));
3853 } else
3854 error = copyin(uaddr, tp, sizeof(*tp));
3855 if (error != 0)
3856 return (error);
3857 if (!timespecvalid_interval(&tp->_timeout))
3858 return (EINVAL);
3859 return (0);
3860 }
3861
3862 static int
umtx_copyin_robust_lists(const void * uaddr,size_t size,struct umtx_robust_lists_params * rb)3863 umtx_copyin_robust_lists(const void *uaddr, size_t size,
3864 struct umtx_robust_lists_params *rb)
3865 {
3866
3867 if (size > sizeof(*rb))
3868 return (EINVAL);
3869 return (copyin(uaddr, rb, size));
3870 }
3871
3872 static int
umtx_copyout_timeout(void * uaddr,size_t sz,struct timespec * tsp)3873 umtx_copyout_timeout(void *uaddr, size_t sz, struct timespec *tsp)
3874 {
3875
3876 /*
3877 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
3878 * and we're only called if sz >= sizeof(timespec) as supplied in the
3879 * copyops.
3880 */
3881 KASSERT(sz >= sizeof(*tsp),
3882 ("umtx_copyops specifies incorrect sizes"));
3883
3884 return (copyout(tsp, uaddr, sizeof(*tsp)));
3885 }
3886
3887 #ifdef COMPAT_FREEBSD10
3888 static int
__umtx_op_lock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3889 __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap,
3890 const struct umtx_copyops *ops)
3891 {
3892 struct timespec *ts, timeout;
3893 int error;
3894
3895 /* Allow a null timespec (wait forever). */
3896 if (uap->uaddr2 == NULL)
3897 ts = NULL;
3898 else {
3899 error = ops->copyin_timeout(uap->uaddr2, &timeout);
3900 if (error != 0)
3901 return (error);
3902 ts = &timeout;
3903 }
3904 #ifdef COMPAT_FREEBSD32
3905 if (ops->compat32)
3906 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3907 #endif
3908 return (do_lock_umtx(td, uap->obj, uap->val, ts));
3909 }
3910
3911 static int
__umtx_op_unlock_umtx(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3912 __umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap,
3913 const struct umtx_copyops *ops)
3914 {
3915 #ifdef COMPAT_FREEBSD32
3916 if (ops->compat32)
3917 return (do_unlock_umtx32(td, uap->obj, uap->val));
3918 #endif
3919 return (do_unlock_umtx(td, uap->obj, uap->val));
3920 }
3921 #endif /* COMPAT_FREEBSD10 */
3922
3923 #if !defined(COMPAT_FREEBSD10)
3924 static int
__umtx_op_unimpl(struct thread * td __unused,struct _umtx_op_args * uap __unused,const struct umtx_copyops * ops __unused)3925 __umtx_op_unimpl(struct thread *td __unused, struct _umtx_op_args *uap __unused,
3926 const struct umtx_copyops *ops __unused)
3927 {
3928 return (EOPNOTSUPP);
3929 }
3930 #endif /* COMPAT_FREEBSD10 */
3931
3932 static int
__umtx_op_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3933 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap,
3934 const struct umtx_copyops *ops)
3935 {
3936 struct _umtx_time timeout, *tm_p;
3937 int error;
3938
3939 if (uap->uaddr2 == NULL)
3940 tm_p = NULL;
3941 else {
3942 error = ops->copyin_umtx_time(
3943 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3944 if (error != 0)
3945 return (error);
3946 tm_p = &timeout;
3947 }
3948 return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0));
3949 }
3950
3951 static int
__umtx_op_wait_uint(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3952 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap,
3953 const struct umtx_copyops *ops)
3954 {
3955 struct _umtx_time timeout, *tm_p;
3956 int error;
3957
3958 if (uap->uaddr2 == NULL)
3959 tm_p = NULL;
3960 else {
3961 error = ops->copyin_umtx_time(
3962 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3963 if (error != 0)
3964 return (error);
3965 tm_p = &timeout;
3966 }
3967 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
3968 }
3969
3970 static int
__umtx_op_wait_uint_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)3971 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap,
3972 const struct umtx_copyops *ops)
3973 {
3974 struct _umtx_time *tm_p, timeout;
3975 int error;
3976
3977 if (uap->uaddr2 == NULL)
3978 tm_p = NULL;
3979 else {
3980 error = ops->copyin_umtx_time(
3981 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3982 if (error != 0)
3983 return (error);
3984 tm_p = &timeout;
3985 }
3986 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
3987 }
3988
3989 static int
__umtx_op_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)3990 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap,
3991 const struct umtx_copyops *ops __unused)
3992 {
3993
3994 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3995 }
3996
3997 #define BATCH_SIZE 128
3998 static int
__umtx_op_nwake_private_native(struct thread * td,struct _umtx_op_args * uap)3999 __umtx_op_nwake_private_native(struct thread *td, struct _umtx_op_args *uap)
4000 {
4001 char *uaddrs[BATCH_SIZE], **upp;
4002 int count, error, i, pos, tocopy;
4003
4004 upp = (char **)uap->obj;
4005 error = 0;
4006 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4007 pos += tocopy) {
4008 tocopy = MIN(count, BATCH_SIZE);
4009 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *));
4010 if (error != 0)
4011 break;
4012 for (i = 0; i < tocopy; ++i) {
4013 kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
4014 }
4015 maybe_yield();
4016 }
4017 return (error);
4018 }
4019
4020 static int
__umtx_op_nwake_private_compat32(struct thread * td,struct _umtx_op_args * uap)4021 __umtx_op_nwake_private_compat32(struct thread *td, struct _umtx_op_args *uap)
4022 {
4023 uint32_t uaddrs[BATCH_SIZE], *upp;
4024 int count, error, i, pos, tocopy;
4025
4026 upp = (uint32_t *)uap->obj;
4027 error = 0;
4028 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4029 pos += tocopy) {
4030 tocopy = MIN(count, BATCH_SIZE);
4031 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t));
4032 if (error != 0)
4033 break;
4034 for (i = 0; i < tocopy; ++i) {
4035 kern_umtx_wake(td, (void *)(uintptr_t)uaddrs[i],
4036 INT_MAX, 1);
4037 }
4038 maybe_yield();
4039 }
4040 return (error);
4041 }
4042
4043 static int
__umtx_op_nwake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4044 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap,
4045 const struct umtx_copyops *ops)
4046 {
4047
4048 if (ops->compat32)
4049 return (__umtx_op_nwake_private_compat32(td, uap));
4050 return (__umtx_op_nwake_private_native(td, uap));
4051 }
4052
4053 static int
__umtx_op_wake_private(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4054 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap,
4055 const struct umtx_copyops *ops __unused)
4056 {
4057
4058 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
4059 }
4060
4061 static int
__umtx_op_lock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4062 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap,
4063 const struct umtx_copyops *ops)
4064 {
4065 struct _umtx_time *tm_p, timeout;
4066 int error;
4067
4068 /* Allow a null timespec (wait forever). */
4069 if (uap->uaddr2 == NULL)
4070 tm_p = NULL;
4071 else {
4072 error = ops->copyin_umtx_time(
4073 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4074 if (error != 0)
4075 return (error);
4076 tm_p = &timeout;
4077 }
4078 return (do_lock_umutex(td, uap->obj, tm_p, 0));
4079 }
4080
4081 static int
__umtx_op_trylock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4082 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap,
4083 const struct umtx_copyops *ops __unused)
4084 {
4085
4086 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
4087 }
4088
4089 static int
__umtx_op_wait_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4090 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap,
4091 const struct umtx_copyops *ops)
4092 {
4093 struct _umtx_time *tm_p, timeout;
4094 int error;
4095
4096 /* Allow a null timespec (wait forever). */
4097 if (uap->uaddr2 == NULL)
4098 tm_p = NULL;
4099 else {
4100 error = ops->copyin_umtx_time(
4101 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4102 if (error != 0)
4103 return (error);
4104 tm_p = &timeout;
4105 }
4106 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
4107 }
4108
4109 static int
__umtx_op_wake_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4110 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap,
4111 const struct umtx_copyops *ops __unused)
4112 {
4113
4114 return (do_wake_umutex(td, uap->obj));
4115 }
4116
4117 static int
__umtx_op_unlock_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4118 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap,
4119 const struct umtx_copyops *ops __unused)
4120 {
4121
4122 return (do_unlock_umutex(td, uap->obj, false));
4123 }
4124
4125 static int
__umtx_op_set_ceiling(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4126 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap,
4127 const struct umtx_copyops *ops __unused)
4128 {
4129
4130 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
4131 }
4132
4133 static int
__umtx_op_cv_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4134 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap,
4135 const struct umtx_copyops *ops)
4136 {
4137 struct timespec *ts, timeout;
4138 int error;
4139
4140 /* Allow a null timespec (wait forever). */
4141 if (uap->uaddr2 == NULL)
4142 ts = NULL;
4143 else {
4144 error = ops->copyin_timeout(uap->uaddr2, &timeout);
4145 if (error != 0)
4146 return (error);
4147 ts = &timeout;
4148 }
4149 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
4150 }
4151
4152 static int
__umtx_op_cv_signal(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4153 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap,
4154 const struct umtx_copyops *ops __unused)
4155 {
4156
4157 return (do_cv_signal(td, uap->obj));
4158 }
4159
4160 static int
__umtx_op_cv_broadcast(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4161 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap,
4162 const struct umtx_copyops *ops __unused)
4163 {
4164
4165 return (do_cv_broadcast(td, uap->obj));
4166 }
4167
4168 static int
__umtx_op_rw_rdlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4169 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap,
4170 const struct umtx_copyops *ops)
4171 {
4172 struct _umtx_time timeout;
4173 int error;
4174
4175 /* Allow a null timespec (wait forever). */
4176 if (uap->uaddr2 == NULL) {
4177 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
4178 } else {
4179 error = ops->copyin_umtx_time(uap->uaddr2,
4180 (size_t)uap->uaddr1, &timeout);
4181 if (error != 0)
4182 return (error);
4183 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
4184 }
4185 return (error);
4186 }
4187
4188 static int
__umtx_op_rw_wrlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4189 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap,
4190 const struct umtx_copyops *ops)
4191 {
4192 struct _umtx_time timeout;
4193 int error;
4194
4195 /* Allow a null timespec (wait forever). */
4196 if (uap->uaddr2 == NULL) {
4197 error = do_rw_wrlock(td, uap->obj, 0);
4198 } else {
4199 error = ops->copyin_umtx_time(uap->uaddr2,
4200 (size_t)uap->uaddr1, &timeout);
4201 if (error != 0)
4202 return (error);
4203
4204 error = do_rw_wrlock(td, uap->obj, &timeout);
4205 }
4206 return (error);
4207 }
4208
4209 static int
__umtx_op_rw_unlock(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4210 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap,
4211 const struct umtx_copyops *ops __unused)
4212 {
4213
4214 return (do_rw_unlock(td, uap->obj));
4215 }
4216
4217 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4218 static int
__umtx_op_sem_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4219 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap,
4220 const struct umtx_copyops *ops)
4221 {
4222 struct _umtx_time *tm_p, timeout;
4223 int error;
4224
4225 /* Allow a null timespec (wait forever). */
4226 if (uap->uaddr2 == NULL)
4227 tm_p = NULL;
4228 else {
4229 error = ops->copyin_umtx_time(
4230 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4231 if (error != 0)
4232 return (error);
4233 tm_p = &timeout;
4234 }
4235 return (do_sem_wait(td, uap->obj, tm_p));
4236 }
4237
4238 static int
__umtx_op_sem_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4239 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap,
4240 const struct umtx_copyops *ops __unused)
4241 {
4242
4243 return (do_sem_wake(td, uap->obj));
4244 }
4245 #endif
4246
4247 static int
__umtx_op_wake2_umutex(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4248 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap,
4249 const struct umtx_copyops *ops __unused)
4250 {
4251
4252 return (do_wake2_umutex(td, uap->obj, uap->val));
4253 }
4254
4255 static int
__umtx_op_sem2_wait(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4256 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap,
4257 const struct umtx_copyops *ops)
4258 {
4259 struct _umtx_time *tm_p, timeout;
4260 size_t uasize;
4261 int error;
4262
4263 /* Allow a null timespec (wait forever). */
4264 if (uap->uaddr2 == NULL) {
4265 uasize = 0;
4266 tm_p = NULL;
4267 } else {
4268 uasize = (size_t)uap->uaddr1;
4269 error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout);
4270 if (error != 0)
4271 return (error);
4272 tm_p = &timeout;
4273 }
4274 error = do_sem2_wait(td, uap->obj, tm_p);
4275 if (error == EINTR && uap->uaddr2 != NULL &&
4276 (timeout._flags & UMTX_ABSTIME) == 0 &&
4277 uasize >= ops->umtx_time_sz + ops->timespec_sz) {
4278 error = ops->copyout_timeout(
4279 (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz),
4280 uasize - ops->umtx_time_sz, &timeout._timeout);
4281 if (error == 0) {
4282 error = EINTR;
4283 }
4284 }
4285
4286 return (error);
4287 }
4288
4289 static int
__umtx_op_sem2_wake(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4290 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap,
4291 const struct umtx_copyops *ops __unused)
4292 {
4293
4294 return (do_sem2_wake(td, uap->obj));
4295 }
4296
4297 #define USHM_OBJ_UMTX(o) \
4298 ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
4299
4300 #define USHMF_LINKED 0x0001
4301 struct umtx_shm_reg {
4302 TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link;
4303 LIST_ENTRY(umtx_shm_reg) ushm_obj_link;
4304 struct umtx_key ushm_key;
4305 struct ucred *ushm_cred;
4306 struct shmfd *ushm_obj;
4307 u_int ushm_refcnt;
4308 u_int ushm_flags;
4309 };
4310
4311 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg);
4312 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg);
4313
4314 static uma_zone_t umtx_shm_reg_zone;
4315 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS];
4316 static struct mtx umtx_shm_lock;
4317 static struct umtx_shm_reg_head umtx_shm_reg_delfree =
4318 TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree);
4319
4320 static void umtx_shm_free_reg(struct umtx_shm_reg *reg);
4321
4322 static void
umtx_shm_reg_delfree_tq(void * context __unused,int pending __unused)4323 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused)
4324 {
4325 struct umtx_shm_reg_head d;
4326 struct umtx_shm_reg *reg, *reg1;
4327
4328 TAILQ_INIT(&d);
4329 mtx_lock(&umtx_shm_lock);
4330 TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link);
4331 mtx_unlock(&umtx_shm_lock);
4332 TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) {
4333 TAILQ_REMOVE(&d, reg, ushm_reg_link);
4334 umtx_shm_free_reg(reg);
4335 }
4336 }
4337
4338 static struct task umtx_shm_reg_delfree_task =
4339 TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL);
4340
4341 /*
4342 * Returns 0 if a SHM with the passed key is found in the registry, in which
4343 * case it is returned through 'oreg'. Otherwise, returns an error among ESRCH
4344 * (no corresponding SHM; ESRCH was chosen for compatibility, ENOENT would have
4345 * been preferable) or EOVERFLOW (there is a corresponding SHM, but reference
4346 * count would overflow, so can't return it), in which case '*oreg' is left
4347 * unchanged.
4348 */
4349 static int
umtx_shm_find_reg_locked(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4350 umtx_shm_find_reg_locked(const struct umtx_key *key,
4351 struct umtx_shm_reg **const oreg)
4352 {
4353 struct umtx_shm_reg *reg;
4354 struct umtx_shm_reg_head *reg_head;
4355
4356 KASSERT(key->shared, ("umtx_p_find_rg: private key"));
4357 mtx_assert(&umtx_shm_lock, MA_OWNED);
4358 reg_head = &umtx_shm_registry[key->hash];
4359 TAILQ_FOREACH(reg, reg_head, ushm_reg_link) {
4360 KASSERT(reg->ushm_key.shared,
4361 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
4362 if (reg->ushm_key.info.shared.object ==
4363 key->info.shared.object &&
4364 reg->ushm_key.info.shared.offset ==
4365 key->info.shared.offset) {
4366 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
4367 KASSERT(reg->ushm_refcnt != 0,
4368 ("reg %p refcnt 0 onlist", reg));
4369 KASSERT((reg->ushm_flags & USHMF_LINKED) != 0,
4370 ("reg %p not linked", reg));
4371 /*
4372 * Don't let overflow happen, just deny a new reference
4373 * (this is additional protection against some reference
4374 * count leak, which is known not to be the case at the
4375 * time of this writing).
4376 */
4377 if (__predict_false(reg->ushm_refcnt == UINT_MAX))
4378 return (EOVERFLOW);
4379 reg->ushm_refcnt++;
4380 *oreg = reg;
4381 return (0);
4382 }
4383 }
4384 return (ESRCH);
4385 }
4386
4387 /*
4388 * Calls umtx_shm_find_reg_unlocked() under the 'umtx_shm_lock'.
4389 */
4390 static int
umtx_shm_find_reg(const struct umtx_key * key,struct umtx_shm_reg ** const oreg)4391 umtx_shm_find_reg(const struct umtx_key *key, struct umtx_shm_reg **const oreg)
4392 {
4393 int error;
4394
4395 mtx_lock(&umtx_shm_lock);
4396 error = umtx_shm_find_reg_locked(key, oreg);
4397 mtx_unlock(&umtx_shm_lock);
4398 return (error);
4399 }
4400
4401 static void
umtx_shm_free_reg(struct umtx_shm_reg * reg)4402 umtx_shm_free_reg(struct umtx_shm_reg *reg)
4403 {
4404
4405 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
4406 crfree(reg->ushm_cred);
4407 shm_drop(reg->ushm_obj);
4408 uma_zfree(umtx_shm_reg_zone, reg);
4409 }
4410
4411 static bool
umtx_shm_unref_reg_locked(struct umtx_shm_reg * reg,bool linked_ref)4412 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool linked_ref)
4413 {
4414 mtx_assert(&umtx_shm_lock, MA_OWNED);
4415 KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg));
4416
4417 if (linked_ref) {
4418 if ((reg->ushm_flags & USHMF_LINKED) == 0)
4419 /*
4420 * The reference tied to USHMF_LINKED has already been
4421 * released concurrently.
4422 */
4423 return (false);
4424
4425 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg,
4426 ushm_reg_link);
4427 LIST_REMOVE(reg, ushm_obj_link);
4428 reg->ushm_flags &= ~USHMF_LINKED;
4429 }
4430
4431 reg->ushm_refcnt--;
4432 return (reg->ushm_refcnt == 0);
4433 }
4434
4435 static void
umtx_shm_unref_reg(struct umtx_shm_reg * reg,bool linked_ref)4436 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool linked_ref)
4437 {
4438 vm_object_t object;
4439 bool dofree;
4440
4441 if (linked_ref) {
4442 /*
4443 * Note: This may be executed multiple times on the same
4444 * shared-memory VM object in presence of concurrent callers
4445 * because 'umtx_shm_lock' is not held all along in umtx_shm()
4446 * and here.
4447 */
4448 object = reg->ushm_obj->shm_object;
4449 VM_OBJECT_WLOCK(object);
4450 vm_object_set_flag(object, OBJ_UMTXDEAD);
4451 VM_OBJECT_WUNLOCK(object);
4452 }
4453 mtx_lock(&umtx_shm_lock);
4454 dofree = umtx_shm_unref_reg_locked(reg, linked_ref);
4455 mtx_unlock(&umtx_shm_lock);
4456 if (dofree)
4457 umtx_shm_free_reg(reg);
4458 }
4459
4460 void
umtx_shm_object_init(vm_object_t object)4461 umtx_shm_object_init(vm_object_t object)
4462 {
4463
4464 LIST_INIT(USHM_OBJ_UMTX(object));
4465 }
4466
4467 void
umtx_shm_object_terminated(vm_object_t object)4468 umtx_shm_object_terminated(vm_object_t object)
4469 {
4470 struct umtx_shm_reg *reg, *reg1;
4471 bool dofree;
4472
4473 if (LIST_EMPTY(USHM_OBJ_UMTX(object)))
4474 return;
4475
4476 dofree = false;
4477 mtx_lock(&umtx_shm_lock);
4478 LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) {
4479 if (umtx_shm_unref_reg_locked(reg, true)) {
4480 TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg,
4481 ushm_reg_link);
4482 dofree = true;
4483 }
4484 }
4485 mtx_unlock(&umtx_shm_lock);
4486 if (dofree)
4487 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task);
4488 }
4489
4490 static int
umtx_shm_create_reg(struct thread * td,const struct umtx_key * key,struct umtx_shm_reg ** res)4491 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
4492 struct umtx_shm_reg **res)
4493 {
4494 struct shmfd *shm;
4495 struct umtx_shm_reg *reg, *reg1;
4496 struct ucred *cred;
4497 int error;
4498
4499 error = umtx_shm_find_reg(key, res);
4500 if (error != ESRCH) {
4501 /*
4502 * Either no error occured, and '*res' was filled, or EOVERFLOW
4503 * was returned, indicating a reference count limit, and we
4504 * won't create a duplicate registration. In both cases, we are
4505 * done.
4506 */
4507 return (error);
4508 }
4509 /* No entry, we will create one. */
4510
4511 cred = td->td_ucred;
4512 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
4513 return (ENOMEM);
4514 shm = shm_alloc(td->td_ucred, O_RDWR, false);
4515 if (shm == NULL) {
4516 chgumtxcnt(cred->cr_ruidinfo, -1, 0);
4517 return (ENOMEM);
4518 }
4519 reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
4520 bcopy(key, ®->ushm_key, sizeof(*key));
4521 reg->ushm_obj = shm;
4522 reg->ushm_cred = crhold(cred);
4523 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
4524 if (error != 0) {
4525 umtx_shm_free_reg(reg);
4526 return (error);
4527 }
4528 mtx_lock(&umtx_shm_lock);
4529 /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */
4530 error = umtx_shm_find_reg_locked(key, ®1);
4531 switch (error) {
4532 case 0:
4533 mtx_unlock(&umtx_shm_lock);
4534 umtx_shm_free_reg(reg);
4535 *res = reg1;
4536 return (0);
4537 case ESRCH:
4538 break;
4539 default:
4540 mtx_unlock(&umtx_shm_lock);
4541 umtx_shm_free_reg(reg);
4542 return (error);
4543 }
4544 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
4545 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
4546 ushm_obj_link);
4547 reg->ushm_flags = USHMF_LINKED;
4548 /*
4549 * This is one reference for the registry and the list of shared
4550 * mutexes referenced by the VM object containing the lock pointer, and
4551 * another for the caller, which it will free after use. So, one of
4552 * these is tied to the presence of USHMF_LINKED.
4553 */
4554 reg->ushm_refcnt = 2;
4555 mtx_unlock(&umtx_shm_lock);
4556 *res = reg;
4557 return (0);
4558 }
4559
4560 static int
umtx_shm_alive(struct thread * td,void * addr)4561 umtx_shm_alive(struct thread *td, void *addr)
4562 {
4563 vm_map_t map;
4564 vm_map_entry_t entry;
4565 vm_object_t object;
4566 vm_pindex_t pindex;
4567 vm_prot_t prot;
4568 int res, ret;
4569 boolean_t wired;
4570
4571 map = &td->td_proc->p_vmspace->vm_map;
4572 res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry,
4573 &object, &pindex, &prot, &wired);
4574 if (res != KERN_SUCCESS)
4575 return (EFAULT);
4576 if (object == NULL)
4577 ret = EINVAL;
4578 else
4579 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
4580 vm_map_lookup_done(map, entry);
4581 return (ret);
4582 }
4583
4584 static void
umtx_shm_init(void)4585 umtx_shm_init(void)
4586 {
4587 int i;
4588
4589 umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg),
4590 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
4591 mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF);
4592 for (i = 0; i < nitems(umtx_shm_registry); i++)
4593 TAILQ_INIT(&umtx_shm_registry[i]);
4594 }
4595
4596 static int
umtx_shm(struct thread * td,void * addr,u_int flags)4597 umtx_shm(struct thread *td, void *addr, u_int flags)
4598 {
4599 struct umtx_key key;
4600 struct umtx_shm_reg *reg;
4601 struct file *fp;
4602 int error, fd;
4603
4604 if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP |
4605 UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1)
4606 return (EINVAL);
4607 if ((flags & UMTX_SHM_ALIVE) != 0)
4608 return (umtx_shm_alive(td, addr));
4609 error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key);
4610 if (error != 0)
4611 return (error);
4612 KASSERT(key.shared == 1, ("non-shared key"));
4613 error = (flags & UMTX_SHM_CREAT) != 0 ?
4614 umtx_shm_create_reg(td, &key, ®) :
4615 umtx_shm_find_reg(&key, ®);
4616 umtx_key_release(&key);
4617 if (error != 0)
4618 return (error);
4619 KASSERT(reg != NULL, ("no reg"));
4620 if ((flags & UMTX_SHM_DESTROY) != 0) {
4621 umtx_shm_unref_reg(reg, true);
4622 } else {
4623 #if 0
4624 #ifdef MAC
4625 error = mac_posixshm_check_open(td->td_ucred,
4626 reg->ushm_obj, FFLAGS(O_RDWR));
4627 if (error == 0)
4628 #endif
4629 error = shm_access(reg->ushm_obj, td->td_ucred,
4630 FFLAGS(O_RDWR));
4631 if (error == 0)
4632 #endif
4633 error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL);
4634 if (error == 0) {
4635 shm_hold(reg->ushm_obj);
4636 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
4637 &shm_ops);
4638 td->td_retval[0] = fd;
4639 fdrop(fp, td);
4640 }
4641 }
4642 umtx_shm_unref_reg(reg, false);
4643 return (error);
4644 }
4645
4646 static int
__umtx_op_shm(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops __unused)4647 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap,
4648 const struct umtx_copyops *ops __unused)
4649 {
4650
4651 return (umtx_shm(td, uap->uaddr1, uap->val));
4652 }
4653
4654 static int
__umtx_op_robust_lists(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4655 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap,
4656 const struct umtx_copyops *ops)
4657 {
4658 struct umtx_robust_lists_params rb;
4659 int error;
4660
4661 if (ops->compat32) {
4662 if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 &&
4663 (td->td_rb_list != 0 || td->td_rbp_list != 0 ||
4664 td->td_rb_inact != 0))
4665 return (EBUSY);
4666 } else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) {
4667 return (EBUSY);
4668 }
4669
4670 bzero(&rb, sizeof(rb));
4671 error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb);
4672 if (error != 0)
4673 return (error);
4674
4675 if (ops->compat32)
4676 td->td_pflags2 |= TDP2_COMPAT32RB;
4677
4678 td->td_rb_list = rb.robust_list_offset;
4679 td->td_rbp_list = rb.robust_priv_list_offset;
4680 td->td_rb_inact = rb.robust_inact_offset;
4681 return (0);
4682 }
4683
4684 static int
__umtx_op_get_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4685 __umtx_op_get_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4686 const struct umtx_copyops *ops)
4687 {
4688 long val;
4689 int error, val1;
4690
4691 val = sbttons(td->td_proc->p_umtx_min_timeout);
4692 if (ops->compat32) {
4693 val1 = (int)val;
4694 error = copyout(&val1, uap->uaddr1, sizeof(val1));
4695 } else {
4696 error = copyout(&val, uap->uaddr1, sizeof(val));
4697 }
4698 return (error);
4699 }
4700
4701 static int
__umtx_op_set_min_timeout(struct thread * td,struct _umtx_op_args * uap,const struct umtx_copyops * ops)4702 __umtx_op_set_min_timeout(struct thread *td, struct _umtx_op_args *uap,
4703 const struct umtx_copyops *ops)
4704 {
4705 if (uap->val < 0)
4706 return (EINVAL);
4707 td->td_proc->p_umtx_min_timeout = nstosbt(uap->val);
4708 return (0);
4709 }
4710
4711 #if defined(__i386__) || defined(__amd64__)
4712 /*
4713 * Provide the standard 32-bit definitions for x86, since native/compat32 use a
4714 * 32-bit time_t there. Other architectures just need the i386 definitions
4715 * along with their standard compat32.
4716 */
4717 struct timespecx32 {
4718 int64_t tv_sec;
4719 int32_t tv_nsec;
4720 };
4721
4722 struct umtx_timex32 {
4723 struct timespecx32 _timeout;
4724 uint32_t _flags;
4725 uint32_t _clockid;
4726 };
4727
4728 #ifndef __i386__
4729 #define timespeci386 timespec32
4730 #define umtx_timei386 umtx_time32
4731 #endif
4732 #else /* !__i386__ && !__amd64__ */
4733 /* 32-bit architectures can emulate i386, so define these almost everywhere. */
4734 struct timespeci386 {
4735 int32_t tv_sec;
4736 int32_t tv_nsec;
4737 };
4738
4739 struct umtx_timei386 {
4740 struct timespeci386 _timeout;
4741 uint32_t _flags;
4742 uint32_t _clockid;
4743 };
4744
4745 #if defined(__LP64__)
4746 #define timespecx32 timespec32
4747 #define umtx_timex32 umtx_time32
4748 #endif
4749 #endif
4750
4751 static int
umtx_copyin_robust_lists32(const void * uaddr,size_t size,struct umtx_robust_lists_params * rbp)4752 umtx_copyin_robust_lists32(const void *uaddr, size_t size,
4753 struct umtx_robust_lists_params *rbp)
4754 {
4755 struct umtx_robust_lists_params_compat32 rb32;
4756 int error;
4757
4758 if (size > sizeof(rb32))
4759 return (EINVAL);
4760 bzero(&rb32, sizeof(rb32));
4761 error = copyin(uaddr, &rb32, size);
4762 if (error != 0)
4763 return (error);
4764 CP(rb32, *rbp, robust_list_offset);
4765 CP(rb32, *rbp, robust_priv_list_offset);
4766 CP(rb32, *rbp, robust_inact_offset);
4767 return (0);
4768 }
4769
4770 #ifndef __i386__
4771 static inline int
umtx_copyin_timeouti386(const void * uaddr,struct timespec * tsp)4772 umtx_copyin_timeouti386(const void *uaddr, struct timespec *tsp)
4773 {
4774 struct timespeci386 ts32;
4775 int error;
4776
4777 error = copyin(uaddr, &ts32, sizeof(ts32));
4778 if (error == 0) {
4779 if (!timespecvalid_interval(&ts32))
4780 error = EINVAL;
4781 else {
4782 CP(ts32, *tsp, tv_sec);
4783 CP(ts32, *tsp, tv_nsec);
4784 }
4785 }
4786 return (error);
4787 }
4788
4789 static inline int
umtx_copyin_umtx_timei386(const void * uaddr,size_t size,struct _umtx_time * tp)4790 umtx_copyin_umtx_timei386(const void *uaddr, size_t size, struct _umtx_time *tp)
4791 {
4792 struct umtx_timei386 t32;
4793 int error;
4794
4795 t32._clockid = CLOCK_REALTIME;
4796 t32._flags = 0;
4797 if (size <= sizeof(t32._timeout))
4798 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4799 else
4800 error = copyin(uaddr, &t32, sizeof(t32));
4801 if (error != 0)
4802 return (error);
4803 if (!timespecvalid_interval(&t32._timeout))
4804 return (EINVAL);
4805 TS_CP(t32, *tp, _timeout);
4806 CP(t32, *tp, _flags);
4807 CP(t32, *tp, _clockid);
4808 return (0);
4809 }
4810
4811 static int
umtx_copyout_timeouti386(void * uaddr,size_t sz,struct timespec * tsp)4812 umtx_copyout_timeouti386(void *uaddr, size_t sz, struct timespec *tsp)
4813 {
4814 struct timespeci386 remain32 = {
4815 .tv_sec = tsp->tv_sec,
4816 .tv_nsec = tsp->tv_nsec,
4817 };
4818
4819 /*
4820 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4821 * and we're only called if sz >= sizeof(timespec) as supplied in the
4822 * copyops.
4823 */
4824 KASSERT(sz >= sizeof(remain32),
4825 ("umtx_copyops specifies incorrect sizes"));
4826
4827 return (copyout(&remain32, uaddr, sizeof(remain32)));
4828 }
4829 #endif /* !__i386__ */
4830
4831 #if defined(__i386__) || defined(__LP64__)
4832 static inline int
umtx_copyin_timeoutx32(const void * uaddr,struct timespec * tsp)4833 umtx_copyin_timeoutx32(const void *uaddr, struct timespec *tsp)
4834 {
4835 struct timespecx32 ts32;
4836 int error;
4837
4838 error = copyin(uaddr, &ts32, sizeof(ts32));
4839 if (error == 0) {
4840 if (!timespecvalid_interval(&ts32))
4841 error = EINVAL;
4842 else {
4843 CP(ts32, *tsp, tv_sec);
4844 CP(ts32, *tsp, tv_nsec);
4845 }
4846 }
4847 return (error);
4848 }
4849
4850 static inline int
umtx_copyin_umtx_timex32(const void * uaddr,size_t size,struct _umtx_time * tp)4851 umtx_copyin_umtx_timex32(const void *uaddr, size_t size, struct _umtx_time *tp)
4852 {
4853 struct umtx_timex32 t32;
4854 int error;
4855
4856 t32._clockid = CLOCK_REALTIME;
4857 t32._flags = 0;
4858 if (size <= sizeof(t32._timeout))
4859 error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout));
4860 else
4861 error = copyin(uaddr, &t32, sizeof(t32));
4862 if (error != 0)
4863 return (error);
4864 if (!timespecvalid_interval(&t32._timeout))
4865 return (EINVAL);
4866 TS_CP(t32, *tp, _timeout);
4867 CP(t32, *tp, _flags);
4868 CP(t32, *tp, _clockid);
4869 return (0);
4870 }
4871
4872 static int
umtx_copyout_timeoutx32(void * uaddr,size_t sz,struct timespec * tsp)4873 umtx_copyout_timeoutx32(void *uaddr, size_t sz, struct timespec *tsp)
4874 {
4875 struct timespecx32 remain32 = {
4876 .tv_sec = tsp->tv_sec,
4877 .tv_nsec = tsp->tv_nsec,
4878 };
4879
4880 /*
4881 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4882 * and we're only called if sz >= sizeof(timespec) as supplied in the
4883 * copyops.
4884 */
4885 KASSERT(sz >= sizeof(remain32),
4886 ("umtx_copyops specifies incorrect sizes"));
4887
4888 return (copyout(&remain32, uaddr, sizeof(remain32)));
4889 }
4890 #endif /* __i386__ || __LP64__ */
4891
4892 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap,
4893 const struct umtx_copyops *umtx_ops);
4894
4895 static const _umtx_op_func op_table[] = {
4896 #ifdef COMPAT_FREEBSD10
4897 [UMTX_OP_LOCK] = __umtx_op_lock_umtx,
4898 [UMTX_OP_UNLOCK] = __umtx_op_unlock_umtx,
4899 #else
4900 [UMTX_OP_LOCK] = __umtx_op_unimpl,
4901 [UMTX_OP_UNLOCK] = __umtx_op_unimpl,
4902 #endif
4903 [UMTX_OP_WAIT] = __umtx_op_wait,
4904 [UMTX_OP_WAKE] = __umtx_op_wake,
4905 [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
4906 [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex,
4907 [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex,
4908 [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling,
4909 [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait,
4910 [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal,
4911 [UMTX_OP_CV_BROADCAST] = __umtx_op_cv_broadcast,
4912 [UMTX_OP_WAIT_UINT] = __umtx_op_wait_uint,
4913 [UMTX_OP_RW_RDLOCK] = __umtx_op_rw_rdlock,
4914 [UMTX_OP_RW_WRLOCK] = __umtx_op_rw_wrlock,
4915 [UMTX_OP_RW_UNLOCK] = __umtx_op_rw_unlock,
4916 [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private,
4917 [UMTX_OP_WAKE_PRIVATE] = __umtx_op_wake_private,
4918 [UMTX_OP_MUTEX_WAIT] = __umtx_op_wait_umutex,
4919 [UMTX_OP_MUTEX_WAKE] = __umtx_op_wake_umutex,
4920 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4921 [UMTX_OP_SEM_WAIT] = __umtx_op_sem_wait,
4922 [UMTX_OP_SEM_WAKE] = __umtx_op_sem_wake,
4923 #else
4924 [UMTX_OP_SEM_WAIT] = __umtx_op_unimpl,
4925 [UMTX_OP_SEM_WAKE] = __umtx_op_unimpl,
4926 #endif
4927 [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private,
4928 [UMTX_OP_MUTEX_WAKE2] = __umtx_op_wake2_umutex,
4929 [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait,
4930 [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake,
4931 [UMTX_OP_SHM] = __umtx_op_shm,
4932 [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists,
4933 [UMTX_OP_GET_MIN_TIMEOUT] = __umtx_op_get_min_timeout,
4934 [UMTX_OP_SET_MIN_TIMEOUT] = __umtx_op_set_min_timeout,
4935 };
4936
4937 static const struct umtx_copyops umtx_native_ops = {
4938 .copyin_timeout = umtx_copyin_timeout,
4939 .copyin_umtx_time = umtx_copyin_umtx_time,
4940 .copyin_robust_lists = umtx_copyin_robust_lists,
4941 .copyout_timeout = umtx_copyout_timeout,
4942 .timespec_sz = sizeof(struct timespec),
4943 .umtx_time_sz = sizeof(struct _umtx_time),
4944 };
4945
4946 #ifndef __i386__
4947 static const struct umtx_copyops umtx_native_opsi386 = {
4948 .copyin_timeout = umtx_copyin_timeouti386,
4949 .copyin_umtx_time = umtx_copyin_umtx_timei386,
4950 .copyin_robust_lists = umtx_copyin_robust_lists32,
4951 .copyout_timeout = umtx_copyout_timeouti386,
4952 .timespec_sz = sizeof(struct timespeci386),
4953 .umtx_time_sz = sizeof(struct umtx_timei386),
4954 .compat32 = true,
4955 };
4956 #endif
4957
4958 #if defined(__i386__) || defined(__LP64__)
4959 /* i386 can emulate other 32-bit archs, too! */
4960 static const struct umtx_copyops umtx_native_opsx32 = {
4961 .copyin_timeout = umtx_copyin_timeoutx32,
4962 .copyin_umtx_time = umtx_copyin_umtx_timex32,
4963 .copyin_robust_lists = umtx_copyin_robust_lists32,
4964 .copyout_timeout = umtx_copyout_timeoutx32,
4965 .timespec_sz = sizeof(struct timespecx32),
4966 .umtx_time_sz = sizeof(struct umtx_timex32),
4967 .compat32 = true,
4968 };
4969
4970 #ifdef COMPAT_FREEBSD32
4971 #ifdef __amd64__
4972 #define umtx_native_ops32 umtx_native_opsi386
4973 #else
4974 #define umtx_native_ops32 umtx_native_opsx32
4975 #endif
4976 #endif /* COMPAT_FREEBSD32 */
4977 #endif /* __i386__ || __LP64__ */
4978
4979 #define UMTX_OP__FLAGS (UMTX_OP__32BIT | UMTX_OP__I386)
4980
4981 static int
kern__umtx_op(struct thread * td,void * obj,int op,unsigned long val,void * uaddr1,void * uaddr2,const struct umtx_copyops * ops)4982 kern__umtx_op(struct thread *td, void *obj, int op, unsigned long val,
4983 void *uaddr1, void *uaddr2, const struct umtx_copyops *ops)
4984 {
4985 struct _umtx_op_args uap = {
4986 .obj = obj,
4987 .op = op & ~UMTX_OP__FLAGS,
4988 .val = val,
4989 .uaddr1 = uaddr1,
4990 .uaddr2 = uaddr2
4991 };
4992
4993 if ((uap.op >= nitems(op_table)))
4994 return (EINVAL);
4995 return ((*op_table[uap.op])(td, &uap, ops));
4996 }
4997
4998 int
sys__umtx_op(struct thread * td,struct _umtx_op_args * uap)4999 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
5000 {
5001 static const struct umtx_copyops *umtx_ops;
5002
5003 umtx_ops = &umtx_native_ops;
5004 #ifdef __LP64__
5005 if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) {
5006 if ((uap->op & UMTX_OP__I386) != 0)
5007 umtx_ops = &umtx_native_opsi386;
5008 else
5009 umtx_ops = &umtx_native_opsx32;
5010 }
5011 #elif !defined(__i386__)
5012 /* We consider UMTX_OP__32BIT a nop on !i386 ILP32. */
5013 if ((uap->op & UMTX_OP__I386) != 0)
5014 umtx_ops = &umtx_native_opsi386;
5015 #else
5016 /* Likewise, UMTX_OP__I386 is a nop on i386. */
5017 if ((uap->op & UMTX_OP__32BIT) != 0)
5018 umtx_ops = &umtx_native_opsx32;
5019 #endif
5020 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5021 uap->uaddr2, umtx_ops));
5022 }
5023
5024 #ifdef COMPAT_FREEBSD32
5025 #ifdef COMPAT_FREEBSD10
5026 int
freebsd10_freebsd32__umtx_lock(struct thread * td,struct freebsd10_freebsd32__umtx_lock_args * uap)5027 freebsd10_freebsd32__umtx_lock(struct thread *td,
5028 struct freebsd10_freebsd32__umtx_lock_args *uap)
5029 {
5030 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
5031 }
5032
5033 int
freebsd10_freebsd32__umtx_unlock(struct thread * td,struct freebsd10_freebsd32__umtx_unlock_args * uap)5034 freebsd10_freebsd32__umtx_unlock(struct thread *td,
5035 struct freebsd10_freebsd32__umtx_unlock_args *uap)
5036 {
5037 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
5038 }
5039 #endif /* COMPAT_FREEBSD10 */
5040
5041 int
freebsd32__umtx_op(struct thread * td,struct freebsd32__umtx_op_args * uap)5042 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap)
5043 {
5044
5045 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5046 uap->uaddr2, &umtx_native_ops32));
5047 }
5048 #endif /* COMPAT_FREEBSD32 */
5049
5050 void
umtx_thread_init(struct thread * td)5051 umtx_thread_init(struct thread *td)
5052 {
5053
5054 td->td_umtxq = umtxq_alloc();
5055 td->td_umtxq->uq_thread = td;
5056 }
5057
5058 void
umtx_thread_fini(struct thread * td)5059 umtx_thread_fini(struct thread *td)
5060 {
5061
5062 umtxq_free(td->td_umtxq);
5063 }
5064
5065 /*
5066 * It will be called when new thread is created, e.g fork().
5067 */
5068 void
umtx_thread_alloc(struct thread * td)5069 umtx_thread_alloc(struct thread *td)
5070 {
5071 struct umtx_q *uq;
5072
5073 uq = td->td_umtxq;
5074 uq->uq_inherited_pri = PRI_MAX;
5075
5076 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
5077 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
5078 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
5079 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
5080 }
5081
5082 /*
5083 * exec() hook.
5084 *
5085 * Clear robust lists for all process' threads, not delaying the
5086 * cleanup to thread exit, since the relevant address space is
5087 * destroyed right now.
5088 */
5089 void
umtx_exec(struct proc * p)5090 umtx_exec(struct proc *p)
5091 {
5092 struct thread *td;
5093
5094 KASSERT(p == curproc, ("need curproc"));
5095 KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
5096 (p->p_flag & P_STOPPED_SINGLE) != 0,
5097 ("curproc must be single-threaded"));
5098 /*
5099 * There is no need to lock the list as only this thread can be
5100 * running.
5101 */
5102 FOREACH_THREAD_IN_PROC(p, td) {
5103 KASSERT(td == curthread ||
5104 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
5105 ("running thread %p %p", p, td));
5106 umtx_thread_cleanup(td);
5107 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
5108 }
5109
5110 p->p_umtx_min_timeout = 0;
5111 }
5112
5113 /*
5114 * thread exit hook.
5115 */
5116 void
umtx_thread_exit(struct thread * td)5117 umtx_thread_exit(struct thread *td)
5118 {
5119
5120 umtx_thread_cleanup(td);
5121 }
5122
5123 static int
umtx_read_uptr(struct thread * td,uintptr_t ptr,uintptr_t * res,bool compat32)5124 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res, bool compat32)
5125 {
5126 u_long res1;
5127 uint32_t res32;
5128 int error;
5129
5130 if (compat32) {
5131 error = fueword32((void *)ptr, &res32);
5132 if (error == 0)
5133 res1 = res32;
5134 } else {
5135 error = fueword((void *)ptr, &res1);
5136 }
5137 if (error == 0)
5138 *res = res1;
5139 else
5140 error = EFAULT;
5141 return (error);
5142 }
5143
5144 static void
umtx_read_rb_list(struct thread * td,struct umutex * m,uintptr_t * rb_list,bool compat32)5145 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list,
5146 bool compat32)
5147 {
5148 struct umutex32 m32;
5149
5150 if (compat32) {
5151 memcpy(&m32, m, sizeof(m32));
5152 *rb_list = m32.m_rb_lnk;
5153 } else {
5154 *rb_list = m->m_rb_lnk;
5155 }
5156 }
5157
5158 static int
umtx_handle_rb(struct thread * td,uintptr_t rbp,uintptr_t * rb_list,bool inact,bool compat32)5159 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact,
5160 bool compat32)
5161 {
5162 struct umutex m;
5163 int error;
5164
5165 KASSERT(td->td_proc == curproc, ("need current vmspace"));
5166 error = copyin((void *)rbp, &m, sizeof(m));
5167 if (error != 0)
5168 return (error);
5169 if (rb_list != NULL)
5170 umtx_read_rb_list(td, &m, rb_list, compat32);
5171 if ((m.m_flags & UMUTEX_ROBUST) == 0)
5172 return (EINVAL);
5173 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
5174 /* inact is cleared after unlock, allow the inconsistency */
5175 return (inact ? 0 : EINVAL);
5176 return (do_unlock_umutex(td, (struct umutex *)rbp, true));
5177 }
5178
5179 static void
umtx_cleanup_rb_list(struct thread * td,uintptr_t rb_list,uintptr_t * rb_inact,const char * name,bool compat32)5180 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact,
5181 const char *name, bool compat32)
5182 {
5183 int error, i;
5184 uintptr_t rbp;
5185 bool inact;
5186
5187 if (rb_list == 0)
5188 return;
5189 error = umtx_read_uptr(td, rb_list, &rbp, compat32);
5190 for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) {
5191 if (rbp == *rb_inact) {
5192 inact = true;
5193 *rb_inact = 0;
5194 } else
5195 inact = false;
5196 error = umtx_handle_rb(td, rbp, &rbp, inact, compat32);
5197 }
5198 if (i == umtx_max_rb && umtx_verbose_rb) {
5199 uprintf("comm %s pid %d: reached umtx %smax rb %d\n",
5200 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
5201 }
5202 if (error != 0 && umtx_verbose_rb) {
5203 uprintf("comm %s pid %d: handling %srb error %d\n",
5204 td->td_proc->p_comm, td->td_proc->p_pid, name, error);
5205 }
5206 }
5207
5208 /*
5209 * Clean up umtx data.
5210 */
5211 static void
umtx_thread_cleanup(struct thread * td)5212 umtx_thread_cleanup(struct thread *td)
5213 {
5214 struct umtx_q *uq;
5215 struct umtx_pi *pi;
5216 uintptr_t rb_inact;
5217 bool compat32;
5218
5219 /*
5220 * Disown pi mutexes.
5221 */
5222 uq = td->td_umtxq;
5223 if (uq != NULL) {
5224 if (uq->uq_inherited_pri != PRI_MAX ||
5225 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
5226 mtx_lock(&umtx_lock);
5227 uq->uq_inherited_pri = PRI_MAX;
5228 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
5229 pi->pi_owner = NULL;
5230 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
5231 }
5232 mtx_unlock(&umtx_lock);
5233 }
5234 sched_lend_user_prio_cond(td, PRI_MAX);
5235 }
5236
5237 compat32 = (td->td_pflags2 & TDP2_COMPAT32RB) != 0;
5238 td->td_pflags2 &= ~TDP2_COMPAT32RB;
5239
5240 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0)
5241 return;
5242
5243 /*
5244 * Handle terminated robust mutexes. Must be done after
5245 * robust pi disown, otherwise unlock could see unowned
5246 * entries.
5247 */
5248 rb_inact = td->td_rb_inact;
5249 if (rb_inact != 0)
5250 (void)umtx_read_uptr(td, rb_inact, &rb_inact, compat32);
5251 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "", compat32);
5252 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ", compat32);
5253 if (rb_inact != 0)
5254 (void)umtx_handle_rb(td, rb_inact, NULL, true, compat32);
5255 }
5256