1 /*
2 * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
3 * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
4 * Copyright (c) 2015, 2016 The FreeBSD Foundation
5 *
6 * All rights reserved.
7 *
8 * Portions of this software were developed by Konstantin Belousov
9 * under sponsorship from the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by John Birrell.
22 * 4. Neither the name of the author nor the names of any co-contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
41
42 #include "namespace.h"
43 #include <stdlib.h>
44 #include <errno.h>
45 #include <string.h>
46 #include <sys/param.h>
47 #include <sys/queue.h>
48 #include <pthread.h>
49 #include <pthread_np.h>
50 #include "un-namespace.h"
51
52 #include "thr_private.h"
53
54 _Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE,
55 "pthread_mutex is too large for off-page");
56
57 /*
58 * For adaptive mutexes, how many times to spin doing trylock2
59 * before entering the kernel to block
60 */
61 #define MUTEX_ADAPTIVE_SPINS 2000
62
63 /*
64 * Prototypes
65 */
66 int __pthread_mutex_consistent(pthread_mutex_t *mutex);
67 int __pthread_mutex_init(pthread_mutex_t * __restrict mutex,
68 const pthread_mutexattr_t * __restrict mutex_attr);
69 int __pthread_mutex_trylock(pthread_mutex_t *mutex);
70 int __pthread_mutex_lock(pthread_mutex_t *mutex);
71 int __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex,
72 const struct timespec * __restrict abstime);
73 int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count);
74 int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
75 int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
76 int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
77 int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count);
78 int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
79
80 static int mutex_self_trylock(pthread_mutex_t);
81 static int mutex_self_lock(pthread_mutex_t,
82 const struct timespec *abstime);
83 static int mutex_unlock_common(struct pthread_mutex *, bool, int *);
84 static int mutex_lock_sleep(struct pthread *, pthread_mutex_t,
85 const struct timespec *);
86 static void mutex_init_robust(struct pthread *curthread);
87 static int mutex_qidx(struct pthread_mutex *m);
88 static bool is_robust_mutex(struct pthread_mutex *m);
89 static bool is_pshared_mutex(struct pthread_mutex *m);
90
91 __weak_reference(__pthread_mutex_init, pthread_mutex_init);
92 __strong_reference(__pthread_mutex_init, _pthread_mutex_init);
93 __weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
94 __strong_reference(__pthread_mutex_lock, _pthread_mutex_lock);
95 __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
96 __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock);
97 __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
98 __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock);
99 __weak_reference(_pthread_mutex_consistent, pthread_mutex_consistent);
100 __strong_reference(_pthread_mutex_consistent, __pthread_mutex_consistent);
101
102 /* Single underscore versions provided for libc internal usage: */
103 /* No difference between libc and application usage of these: */
104 __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
105 __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
106
107 __weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling);
108 __weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling);
109
110 __weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np);
111 __strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np);
112 __weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np);
113
114 __weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np);
115 __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np);
116 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
117 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
118
119 static void
mutex_init_link(struct pthread_mutex * m)120 mutex_init_link(struct pthread_mutex *m)
121 {
122
123 #if defined(_PTHREADS_INVARIANTS)
124 m->m_qe.tqe_prev = NULL;
125 m->m_qe.tqe_next = NULL;
126 m->m_pqe.tqe_prev = NULL;
127 m->m_pqe.tqe_next = NULL;
128 #endif
129 }
130
131 static void
mutex_assert_is_owned(struct pthread_mutex * m __unused)132 mutex_assert_is_owned(struct pthread_mutex *m __unused)
133 {
134
135 #if defined(_PTHREADS_INVARIANTS)
136 if (__predict_false(m->m_qe.tqe_prev == NULL))
137 PANIC("mutex %p own %#x is not on list %p %p",
138 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
139 #endif
140 }
141
142 static void
mutex_assert_not_owned(struct pthread * curthread __unused,struct pthread_mutex * m __unused)143 mutex_assert_not_owned(struct pthread *curthread __unused,
144 struct pthread_mutex *m __unused)
145 {
146
147 #if defined(_PTHREADS_INVARIANTS)
148 if (__predict_false(m->m_qe.tqe_prev != NULL ||
149 m->m_qe.tqe_next != NULL))
150 PANIC("mutex %p own %#x is on list %p %p",
151 m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
152 if (__predict_false(is_robust_mutex(m) &&
153 (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||
154 (is_pshared_mutex(m) && curthread->robust_list ==
155 (uintptr_t)&m->m_lock) ||
156 (!is_pshared_mutex(m) && curthread->priv_robust_list ==
157 (uintptr_t)&m->m_lock))))
158 PANIC(
159 "mutex %p own %#x is on robust linkage %p %p head %p phead %p",
160 m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,
161 m->m_rb_prev, (void *)curthread->robust_list,
162 (void *)curthread->priv_robust_list);
163 #endif
164 }
165
166 static bool
is_pshared_mutex(struct pthread_mutex * m)167 is_pshared_mutex(struct pthread_mutex *m)
168 {
169
170 return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
171 }
172
173 static bool
is_robust_mutex(struct pthread_mutex * m)174 is_robust_mutex(struct pthread_mutex *m)
175 {
176
177 return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0);
178 }
179
180 int
_mutex_enter_robust(struct pthread * curthread,struct pthread_mutex * m)181 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)
182 {
183
184 #if defined(_PTHREADS_INVARIANTS)
185 if (__predict_false(curthread->inact_mtx != 0))
186 PANIC("inact_mtx enter");
187 #endif
188 if (!is_robust_mutex(m))
189 return (0);
190
191 mutex_init_robust(curthread);
192 curthread->inact_mtx = (uintptr_t)&m->m_lock;
193 return (1);
194 }
195
196 void
_mutex_leave_robust(struct pthread * curthread,struct pthread_mutex * m __unused)197 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)
198 {
199
200 #if defined(_PTHREADS_INVARIANTS)
201 if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))
202 PANIC("inact_mtx leave");
203 #endif
204 curthread->inact_mtx = 0;
205 }
206
207 static int
mutex_check_attr(const struct pthread_mutex_attr * attr)208 mutex_check_attr(const struct pthread_mutex_attr *attr)
209 {
210
211 if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
212 attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
213 return (EINVAL);
214 if (attr->m_protocol < PTHREAD_PRIO_NONE ||
215 attr->m_protocol > PTHREAD_PRIO_PROTECT)
216 return (EINVAL);
217 return (0);
218 }
219
220 static void
mutex_init_robust(struct pthread * curthread)221 mutex_init_robust(struct pthread *curthread)
222 {
223 struct umtx_robust_lists_params rb;
224
225 if (curthread == NULL)
226 curthread = _get_curthread();
227 if (curthread->robust_inited)
228 return;
229 rb.robust_list_offset = (uintptr_t)&curthread->robust_list;
230 rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;
231 rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;
232 _umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL);
233 curthread->robust_inited = 1;
234 }
235
236 static void
mutex_init_body(struct pthread_mutex * pmutex,const struct pthread_mutex_attr * attr)237 mutex_init_body(struct pthread_mutex *pmutex,
238 const struct pthread_mutex_attr *attr)
239 {
240
241 pmutex->m_flags = attr->m_type;
242 pmutex->m_count = 0;
243 pmutex->m_spinloops = 0;
244 pmutex->m_yieldloops = 0;
245 mutex_init_link(pmutex);
246 switch (attr->m_protocol) {
247 case PTHREAD_PRIO_NONE:
248 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
249 pmutex->m_lock.m_flags = 0;
250 break;
251 case PTHREAD_PRIO_INHERIT:
252 pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
253 pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT;
254 break;
255 case PTHREAD_PRIO_PROTECT:
256 pmutex->m_lock.m_owner = UMUTEX_CONTESTED;
257 pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT;
258 pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
259 break;
260 }
261 if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
262 pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
263 if (attr->m_robust == PTHREAD_MUTEX_ROBUST) {
264 mutex_init_robust(NULL);
265 pmutex->m_lock.m_flags |= UMUTEX_ROBUST;
266 }
267 if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
268 pmutex->m_spinloops =
269 _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
270 pmutex->m_yieldloops = _thr_yieldloops;
271 }
272 }
273
274 static int
mutex_init(pthread_mutex_t * mutex,const struct pthread_mutex_attr * mutex_attr,void * (calloc_cb)(size_t,size_t))275 mutex_init(pthread_mutex_t *mutex,
276 const struct pthread_mutex_attr *mutex_attr,
277 void *(calloc_cb)(size_t, size_t))
278 {
279 const struct pthread_mutex_attr *attr;
280 struct pthread_mutex *pmutex;
281 int error;
282
283 if (mutex_attr == NULL) {
284 attr = &_pthread_mutexattr_default;
285 } else {
286 attr = mutex_attr;
287 error = mutex_check_attr(attr);
288 if (error != 0)
289 return (error);
290 }
291 if ((pmutex = (pthread_mutex_t)
292 calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
293 return (ENOMEM);
294 mutex_init_body(pmutex, attr);
295 *mutex = pmutex;
296 return (0);
297 }
298
299 static int
init_static(struct pthread * thread,pthread_mutex_t * mutex)300 init_static(struct pthread *thread, pthread_mutex_t *mutex)
301 {
302 int ret;
303
304 THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
305
306 if (*mutex == THR_MUTEX_INITIALIZER)
307 ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
308 else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
309 ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
310 calloc);
311 else
312 ret = 0;
313 THR_LOCK_RELEASE(thread, &_mutex_static_lock);
314
315 return (ret);
316 }
317
318 static void
set_inherited_priority(struct pthread * curthread,struct pthread_mutex * m)319 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
320 {
321 struct pthread_mutex *m2;
322
323 m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);
324 if (m2 != NULL)
325 m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
326 else
327 m->m_lock.m_ceilings[1] = -1;
328 }
329
330 static void
shared_mutex_init(struct pthread_mutex * pmtx,const struct pthread_mutex_attr * mutex_attr)331 shared_mutex_init(struct pthread_mutex *pmtx, const struct
332 pthread_mutex_attr *mutex_attr)
333 {
334 static const struct pthread_mutex_attr foobar_mutex_attr = {
335 .m_type = PTHREAD_MUTEX_DEFAULT,
336 .m_protocol = PTHREAD_PRIO_NONE,
337 .m_ceiling = 0,
338 .m_pshared = PTHREAD_PROCESS_SHARED,
339 .m_robust = PTHREAD_MUTEX_STALLED,
340 };
341 bool done;
342
343 /*
344 * Hack to allow multiple pthread_mutex_init() calls on the
345 * same process-shared mutex. We rely on kernel allocating
346 * zeroed offpage for the mutex, i.e. the
347 * PMUTEX_INITSTAGE_ALLOC value must be zero.
348 */
349 for (done = false; !done;) {
350 switch (pmtx->m_ps) {
351 case PMUTEX_INITSTAGE_DONE:
352 atomic_thread_fence_acq();
353 done = true;
354 break;
355 case PMUTEX_INITSTAGE_ALLOC:
356 if (atomic_cmpset_int(&pmtx->m_ps,
357 PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) {
358 if (mutex_attr == NULL)
359 mutex_attr = &foobar_mutex_attr;
360 mutex_init_body(pmtx, mutex_attr);
361 atomic_store_rel_int(&pmtx->m_ps,
362 PMUTEX_INITSTAGE_DONE);
363 done = true;
364 }
365 break;
366 case PMUTEX_INITSTAGE_BUSY:
367 _pthread_yield();
368 break;
369 default:
370 PANIC("corrupted offpage");
371 break;
372 }
373 }
374 }
375
376 int
__pthread_mutex_init(pthread_mutex_t * __restrict mutex,const pthread_mutexattr_t * __restrict mutex_attr)377 __pthread_mutex_init(pthread_mutex_t * __restrict mutex,
378 const pthread_mutexattr_t * __restrict mutex_attr)
379 {
380 struct pthread_mutex *pmtx;
381 int ret;
382
383 _thr_check_init();
384
385 if (mutex_attr != NULL) {
386 ret = mutex_check_attr(*mutex_attr);
387 if (ret != 0)
388 return (ret);
389 }
390 if (mutex_attr == NULL ||
391 (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
392 return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
393 calloc));
394 }
395 pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1);
396 if (pmtx == NULL)
397 return (EFAULT);
398 *mutex = THR_PSHARED_PTR;
399 shared_mutex_init(pmtx, *mutex_attr);
400 return (0);
401 }
402
403 /* This function is used internally by malloc. */
404 int
_pthread_mutex_init_calloc_cb(pthread_mutex_t * mutex,void * (calloc_cb)(size_t,size_t))405 _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
406 void *(calloc_cb)(size_t, size_t))
407 {
408 static const struct pthread_mutex_attr attr = {
409 .m_type = PTHREAD_MUTEX_NORMAL,
410 .m_protocol = PTHREAD_PRIO_NONE,
411 .m_ceiling = 0,
412 .m_pshared = PTHREAD_PROCESS_PRIVATE,
413 .m_robust = PTHREAD_MUTEX_STALLED,
414 };
415 int ret;
416
417 ret = mutex_init(mutex, &attr, calloc_cb);
418 if (ret == 0)
419 (*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE;
420 return (ret);
421 }
422
423 /*
424 * Fix mutex ownership for child process.
425 *
426 * Process private mutex ownership is transmitted from the forking
427 * thread to the child process.
428 *
429 * Process shared mutex should not be inherited because owner is
430 * forking thread which is in parent process, they are removed from
431 * the owned mutex list.
432 */
433 static void
queue_fork(struct pthread * curthread,struct mutex_queue * q,struct mutex_queue * qp,uint bit)434 queue_fork(struct pthread *curthread, struct mutex_queue *q,
435 struct mutex_queue *qp, uint bit)
436 {
437 struct pthread_mutex *m;
438
439 TAILQ_INIT(q);
440 TAILQ_FOREACH(m, qp, m_pqe) {
441 TAILQ_INSERT_TAIL(q, m, m_qe);
442 m->m_lock.m_owner = TID(curthread) | bit;
443 }
444 }
445
446 void
_mutex_fork(struct pthread * curthread)447 _mutex_fork(struct pthread *curthread)
448 {
449
450 queue_fork(curthread, &curthread->mq[TMQ_NORM],
451 &curthread->mq[TMQ_NORM_PRIV], 0);
452 queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
453 &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
454 queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],
455 &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);
456 curthread->robust_list = 0;
457 }
458
459 int
_pthread_mutex_destroy(pthread_mutex_t * mutex)460 _pthread_mutex_destroy(pthread_mutex_t *mutex)
461 {
462 pthread_mutex_t m, m1;
463 int ret;
464
465 m = *mutex;
466 if (m < THR_MUTEX_DESTROYED) {
467 ret = 0;
468 } else if (m == THR_MUTEX_DESTROYED) {
469 ret = EINVAL;
470 } else {
471 if (m == THR_PSHARED_PTR) {
472 m1 = __thr_pshared_offpage(mutex, 0);
473 if (m1 != NULL) {
474 if ((uint32_t)m1->m_lock.m_owner !=
475 UMUTEX_RB_OWNERDEAD) {
476 mutex_assert_not_owned(
477 _get_curthread(), m1);
478 }
479 __thr_pshared_destroy(mutex);
480 }
481 *mutex = THR_MUTEX_DESTROYED;
482 return (0);
483 }
484 if (PMUTEX_OWNER_ID(m) != 0 &&
485 (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) {
486 ret = EBUSY;
487 } else {
488 *mutex = THR_MUTEX_DESTROYED;
489 mutex_assert_not_owned(_get_curthread(), m);
490 free(m);
491 ret = 0;
492 }
493 }
494
495 return (ret);
496 }
497
498 static int
mutex_qidx(struct pthread_mutex * m)499 mutex_qidx(struct pthread_mutex *m)
500 {
501
502 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
503 return (TMQ_NORM);
504 return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP);
505 }
506
507 /*
508 * Both enqueue_mutex() and dequeue_mutex() operate on the
509 * thread-private linkage of the locked mutexes and on the robust
510 * linkage.
511 *
512 * Robust list, as seen by kernel, must be consistent even in the case
513 * of thread termination at arbitrary moment. Since either enqueue or
514 * dequeue for list walked by kernel consists of rewriting a single
515 * forward pointer, it is safe. On the other hand, rewrite of the
516 * back pointer is not atomic WRT the forward one, but kernel does not
517 * care.
518 */
519 static void
enqueue_mutex(struct pthread * curthread,struct pthread_mutex * m,int error)520 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,
521 int error)
522 {
523 struct pthread_mutex *m1;
524 uintptr_t *rl;
525 int qidx;
526
527 /* Add to the list of owned mutexes: */
528 if (error != EOWNERDEAD)
529 mutex_assert_not_owned(curthread, m);
530 qidx = mutex_qidx(m);
531 TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
532 if (!is_pshared_mutex(m))
533 TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
534 if (is_robust_mutex(m)) {
535 rl = is_pshared_mutex(m) ? &curthread->robust_list :
536 &curthread->priv_robust_list;
537 m->m_rb_prev = NULL;
538 if (*rl != 0) {
539 m1 = __containerof((void *)*rl,
540 struct pthread_mutex, m_lock);
541 m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock;
542 m1->m_rb_prev = m;
543 } else {
544 m1 = NULL;
545 m->m_lock.m_rb_lnk = 0;
546 }
547 *rl = (uintptr_t)&m->m_lock;
548 }
549 }
550
551 static void
dequeue_mutex(struct pthread * curthread,struct pthread_mutex * m)552 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
553 {
554 struct pthread_mutex *mp, *mn;
555 int qidx;
556
557 mutex_assert_is_owned(m);
558 qidx = mutex_qidx(m);
559 if (is_robust_mutex(m)) {
560 mp = m->m_rb_prev;
561 if (mp == NULL) {
562 if (is_pshared_mutex(m)) {
563 curthread->robust_list = m->m_lock.m_rb_lnk;
564 } else {
565 curthread->priv_robust_list =
566 m->m_lock.m_rb_lnk;
567 }
568 } else {
569 mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk;
570 }
571 if (m->m_lock.m_rb_lnk != 0) {
572 mn = __containerof((void *)m->m_lock.m_rb_lnk,
573 struct pthread_mutex, m_lock);
574 mn->m_rb_prev = m->m_rb_prev;
575 }
576 m->m_lock.m_rb_lnk = 0;
577 m->m_rb_prev = NULL;
578 }
579 TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
580 if (!is_pshared_mutex(m))
581 TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
582 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
583 set_inherited_priority(curthread, m);
584 mutex_init_link(m);
585 }
586
587 static int
check_and_init_mutex(pthread_mutex_t * mutex,struct pthread_mutex ** m)588 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
589 {
590 int ret;
591
592 *m = *mutex;
593 ret = 0;
594 if (*m == THR_PSHARED_PTR) {
595 *m = __thr_pshared_offpage(mutex, 0);
596 if (*m == NULL)
597 ret = EINVAL;
598 else
599 shared_mutex_init(*m, NULL);
600 } else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
601 if (*m == THR_MUTEX_DESTROYED) {
602 ret = EINVAL;
603 } else {
604 ret = init_static(_get_curthread(), mutex);
605 if (ret == 0)
606 *m = *mutex;
607 }
608 }
609 return (ret);
610 }
611
612 int
__pthread_mutex_trylock(pthread_mutex_t * mutex)613 __pthread_mutex_trylock(pthread_mutex_t *mutex)
614 {
615 struct pthread *curthread;
616 struct pthread_mutex *m;
617 uint32_t id;
618 int ret, robust;
619
620 ret = check_and_init_mutex(mutex, &m);
621 if (ret != 0)
622 return (ret);
623 curthread = _get_curthread();
624 id = TID(curthread);
625 if (m->m_flags & PMUTEX_FLAG_PRIVATE)
626 THR_CRITICAL_ENTER(curthread);
627 robust = _mutex_enter_robust(curthread, m);
628 ret = _thr_umutex_trylock(&m->m_lock, id);
629 if (__predict_true(ret == 0) || ret == EOWNERDEAD) {
630 enqueue_mutex(curthread, m, ret);
631 if (ret == EOWNERDEAD)
632 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
633 } else if (PMUTEX_OWNER_ID(m) == id) {
634 ret = mutex_self_trylock(m);
635 } /* else {} */
636 if (robust)
637 _mutex_leave_robust(curthread, m);
638 if (ret != 0 && ret != EOWNERDEAD &&
639 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0)
640 THR_CRITICAL_LEAVE(curthread);
641 return (ret);
642 }
643
644 static int
mutex_lock_sleep(struct pthread * curthread,struct pthread_mutex * m,const struct timespec * abstime)645 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
646 const struct timespec *abstime)
647 {
648 uint32_t id, owner;
649 int count, ret;
650
651 id = TID(curthread);
652 if (PMUTEX_OWNER_ID(m) == id)
653 return (mutex_self_lock(m, abstime));
654
655 /*
656 * For adaptive mutexes, spin for a bit in the expectation
657 * that if the application requests this mutex type then
658 * the lock is likely to be released quickly and it is
659 * faster than entering the kernel
660 */
661 if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT |
662 UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0))
663 goto sleep_in_kernel;
664
665 if (!_thr_is_smp)
666 goto yield_loop;
667
668 count = m->m_spinloops;
669 while (count--) {
670 owner = m->m_lock.m_owner;
671 if ((owner & ~UMUTEX_CONTESTED) == 0) {
672 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
673 id | owner)) {
674 ret = 0;
675 goto done;
676 }
677 }
678 CPU_SPINWAIT;
679 }
680
681 yield_loop:
682 count = m->m_yieldloops;
683 while (count--) {
684 _sched_yield();
685 owner = m->m_lock.m_owner;
686 if ((owner & ~UMUTEX_CONTESTED) == 0) {
687 if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
688 id | owner)) {
689 ret = 0;
690 goto done;
691 }
692 }
693 }
694
695 sleep_in_kernel:
696 if (abstime == NULL)
697 ret = __thr_umutex_lock(&m->m_lock, id);
698 else if (__predict_false(abstime->tv_nsec < 0 ||
699 abstime->tv_nsec >= 1000000000))
700 ret = EINVAL;
701 else
702 ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
703 done:
704 if (ret == 0 || ret == EOWNERDEAD) {
705 enqueue_mutex(curthread, m, ret);
706 if (ret == EOWNERDEAD)
707 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
708 }
709 return (ret);
710 }
711
712 static inline int
mutex_lock_common(struct pthread_mutex * m,const struct timespec * abstime,bool cvattach,bool rb_onlist)713 mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime,
714 bool cvattach, bool rb_onlist)
715 {
716 struct pthread *curthread;
717 int ret, robust;
718
719 robust = 0; /* pacify gcc */
720 curthread = _get_curthread();
721 if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
722 THR_CRITICAL_ENTER(curthread);
723 if (!rb_onlist)
724 robust = _mutex_enter_robust(curthread, m);
725 ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));
726 if (ret == 0 || ret == EOWNERDEAD) {
727 enqueue_mutex(curthread, m, ret);
728 if (ret == EOWNERDEAD)
729 m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
730 } else {
731 ret = mutex_lock_sleep(curthread, m, abstime);
732 }
733 if (!rb_onlist && robust)
734 _mutex_leave_robust(curthread, m);
735 if (ret != 0 && ret != EOWNERDEAD &&
736 (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach)
737 THR_CRITICAL_LEAVE(curthread);
738 return (ret);
739 }
740
741 int
__pthread_mutex_lock(pthread_mutex_t * mutex)742 __pthread_mutex_lock(pthread_mutex_t *mutex)
743 {
744 struct pthread_mutex *m;
745 int ret;
746
747 _thr_check_init();
748 ret = check_and_init_mutex(mutex, &m);
749 if (ret == 0)
750 ret = mutex_lock_common(m, NULL, false, false);
751 return (ret);
752 }
753
754 int
__pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex,const struct timespec * __restrict abstime)755 __pthread_mutex_timedlock(pthread_mutex_t * __restrict mutex,
756 const struct timespec * __restrict abstime)
757 {
758 struct pthread_mutex *m;
759 int ret;
760
761 _thr_check_init();
762 ret = check_and_init_mutex(mutex, &m);
763 if (ret == 0)
764 ret = mutex_lock_common(m, abstime, false, false);
765 return (ret);
766 }
767
768 int
_pthread_mutex_unlock(pthread_mutex_t * mutex)769 _pthread_mutex_unlock(pthread_mutex_t *mutex)
770 {
771 struct pthread_mutex *mp;
772
773 if (*mutex == THR_PSHARED_PTR) {
774 mp = __thr_pshared_offpage(mutex, 0);
775 if (mp == NULL)
776 return (EINVAL);
777 shared_mutex_init(mp, NULL);
778 } else {
779 mp = *mutex;
780 }
781 return (mutex_unlock_common(mp, false, NULL));
782 }
783
784 int
_mutex_cv_lock(struct pthread_mutex * m,int count,bool rb_onlist)785 _mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist)
786 {
787 int error;
788
789 error = mutex_lock_common(m, NULL, true, rb_onlist);
790 if (error == 0 || error == EOWNERDEAD)
791 m->m_count = count;
792 return (error);
793 }
794
795 int
_mutex_cv_unlock(struct pthread_mutex * m,int * count,int * defer)796 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer)
797 {
798
799 /*
800 * Clear the count in case this is a recursive mutex.
801 */
802 *count = m->m_count;
803 m->m_count = 0;
804 (void)mutex_unlock_common(m, true, defer);
805 return (0);
806 }
807
808 int
_mutex_cv_attach(struct pthread_mutex * m,int count)809 _mutex_cv_attach(struct pthread_mutex *m, int count)
810 {
811 struct pthread *curthread;
812
813 curthread = _get_curthread();
814 enqueue_mutex(curthread, m, 0);
815 m->m_count = count;
816 return (0);
817 }
818
819 int
_mutex_cv_detach(struct pthread_mutex * mp,int * recurse)820 _mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
821 {
822 struct pthread *curthread;
823 int deferred, error;
824
825 curthread = _get_curthread();
826 if ((error = _mutex_owned(curthread, mp)) != 0)
827 return (error);
828
829 /*
830 * Clear the count in case this is a recursive mutex.
831 */
832 *recurse = mp->m_count;
833 mp->m_count = 0;
834 dequeue_mutex(curthread, mp);
835
836 /* Will this happen in real-world ? */
837 if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
838 deferred = 1;
839 mp->m_flags &= ~PMUTEX_FLAG_DEFERRED;
840 } else
841 deferred = 0;
842
843 if (deferred) {
844 _thr_wake_all(curthread->defer_waiters,
845 curthread->nwaiter_defer);
846 curthread->nwaiter_defer = 0;
847 }
848 return (0);
849 }
850
851 static int
mutex_self_trylock(struct pthread_mutex * m)852 mutex_self_trylock(struct pthread_mutex *m)
853 {
854 int ret;
855
856 switch (PMUTEX_TYPE(m->m_flags)) {
857 case PTHREAD_MUTEX_ERRORCHECK:
858 case PTHREAD_MUTEX_NORMAL:
859 case PTHREAD_MUTEX_ADAPTIVE_NP:
860 ret = EBUSY;
861 break;
862
863 case PTHREAD_MUTEX_RECURSIVE:
864 /* Increment the lock count: */
865 if (m->m_count + 1 > 0) {
866 m->m_count++;
867 ret = 0;
868 } else
869 ret = EAGAIN;
870 break;
871
872 default:
873 /* Trap invalid mutex types; */
874 ret = EINVAL;
875 }
876
877 return (ret);
878 }
879
880 static int
mutex_self_lock(struct pthread_mutex * m,const struct timespec * abstime)881 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)
882 {
883 struct timespec ts1, ts2;
884 int ret;
885
886 switch (PMUTEX_TYPE(m->m_flags)) {
887 case PTHREAD_MUTEX_ERRORCHECK:
888 case PTHREAD_MUTEX_ADAPTIVE_NP:
889 if (abstime) {
890 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
891 abstime->tv_nsec >= 1000000000) {
892 ret = EINVAL;
893 } else {
894 clock_gettime(CLOCK_REALTIME, &ts1);
895 TIMESPEC_SUB(&ts2, abstime, &ts1);
896 __sys_nanosleep(&ts2, NULL);
897 ret = ETIMEDOUT;
898 }
899 } else {
900 /*
901 * POSIX specifies that mutexes should return
902 * EDEADLK if a recursive lock is detected.
903 */
904 ret = EDEADLK;
905 }
906 break;
907
908 case PTHREAD_MUTEX_NORMAL:
909 /*
910 * What SS2 define as a 'normal' mutex. Intentionally
911 * deadlock on attempts to get a lock you already own.
912 */
913 ret = 0;
914 if (abstime) {
915 if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
916 abstime->tv_nsec >= 1000000000) {
917 ret = EINVAL;
918 } else {
919 clock_gettime(CLOCK_REALTIME, &ts1);
920 TIMESPEC_SUB(&ts2, abstime, &ts1);
921 __sys_nanosleep(&ts2, NULL);
922 ret = ETIMEDOUT;
923 }
924 } else {
925 ts1.tv_sec = 30;
926 ts1.tv_nsec = 0;
927 for (;;)
928 __sys_nanosleep(&ts1, NULL);
929 }
930 break;
931
932 case PTHREAD_MUTEX_RECURSIVE:
933 /* Increment the lock count: */
934 if (m->m_count + 1 > 0) {
935 m->m_count++;
936 ret = 0;
937 } else
938 ret = EAGAIN;
939 break;
940
941 default:
942 /* Trap invalid mutex types; */
943 ret = EINVAL;
944 }
945
946 return (ret);
947 }
948
949 static int
mutex_unlock_common(struct pthread_mutex * m,bool cv,int * mtx_defer)950 mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer)
951 {
952 struct pthread *curthread;
953 uint32_t id;
954 int deferred, error, private, robust;
955
956 if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
957 if (m == THR_MUTEX_DESTROYED)
958 return (EINVAL);
959 return (EPERM);
960 }
961
962 curthread = _get_curthread();
963 id = TID(curthread);
964
965 /*
966 * Check if the running thread is not the owner of the mutex.
967 */
968 if (__predict_false(PMUTEX_OWNER_ID(m) != id))
969 return (EPERM);
970
971 error = 0;
972 private = (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0;
973 if (__predict_false(PMUTEX_TYPE(m->m_flags) ==
974 PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) {
975 m->m_count--;
976 } else {
977 if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
978 deferred = 1;
979 m->m_flags &= ~PMUTEX_FLAG_DEFERRED;
980 } else
981 deferred = 0;
982
983 robust = _mutex_enter_robust(curthread, m);
984 dequeue_mutex(curthread, m);
985 error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
986 if (deferred) {
987 if (mtx_defer == NULL) {
988 _thr_wake_all(curthread->defer_waiters,
989 curthread->nwaiter_defer);
990 curthread->nwaiter_defer = 0;
991 } else
992 *mtx_defer = 1;
993 }
994 if (robust)
995 _mutex_leave_robust(curthread, m);
996 }
997 if (!cv && private)
998 THR_CRITICAL_LEAVE(curthread);
999 return (error);
1000 }
1001
1002 int
_pthread_mutex_getprioceiling(const pthread_mutex_t * __restrict mutex,int * __restrict prioceiling)1003 _pthread_mutex_getprioceiling(const pthread_mutex_t * __restrict mutex,
1004 int * __restrict prioceiling)
1005 {
1006 struct pthread_mutex *m;
1007
1008 if (*mutex == THR_PSHARED_PTR) {
1009 m = __thr_pshared_offpage(__DECONST(void *, mutex), 0);
1010 if (m == NULL)
1011 return (EINVAL);
1012 shared_mutex_init(m, NULL);
1013 } else {
1014 m = *mutex;
1015 if (m <= THR_MUTEX_DESTROYED)
1016 return (EINVAL);
1017 }
1018 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
1019 return (EINVAL);
1020 *prioceiling = m->m_lock.m_ceilings[0];
1021 return (0);
1022 }
1023
1024 int
_pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex,int ceiling,int * __restrict old_ceiling)1025 _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex,
1026 int ceiling, int * __restrict old_ceiling)
1027 {
1028 struct pthread *curthread;
1029 struct pthread_mutex *m, *m1, *m2;
1030 struct mutex_queue *q, *qp;
1031 int qidx, ret;
1032
1033 if (*mutex == THR_PSHARED_PTR) {
1034 m = __thr_pshared_offpage(mutex, 0);
1035 if (m == NULL)
1036 return (EINVAL);
1037 shared_mutex_init(m, NULL);
1038 } else {
1039 m = *mutex;
1040 if (m <= THR_MUTEX_DESTROYED)
1041 return (EINVAL);
1042 }
1043 if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
1044 return (EINVAL);
1045
1046 ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
1047 if (ret != 0)
1048 return (ret);
1049
1050 curthread = _get_curthread();
1051 if (PMUTEX_OWNER_ID(m) == TID(curthread)) {
1052 mutex_assert_is_owned(m);
1053 m1 = TAILQ_PREV(m, mutex_queue, m_qe);
1054 m2 = TAILQ_NEXT(m, m_qe);
1055 if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
1056 (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
1057 qidx = mutex_qidx(m);
1058 q = &curthread->mq[qidx];
1059 qp = &curthread->mq[qidx + 1];
1060 TAILQ_REMOVE(q, m, m_qe);
1061 if (!is_pshared_mutex(m))
1062 TAILQ_REMOVE(qp, m, m_pqe);
1063 TAILQ_FOREACH(m2, q, m_qe) {
1064 if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
1065 TAILQ_INSERT_BEFORE(m2, m, m_qe);
1066 if (!is_pshared_mutex(m)) {
1067 while (m2 != NULL &&
1068 is_pshared_mutex(m2)) {
1069 m2 = TAILQ_PREV(m2,
1070 mutex_queue, m_qe);
1071 }
1072 if (m2 == NULL) {
1073 TAILQ_INSERT_HEAD(qp,
1074 m, m_pqe);
1075 } else {
1076 TAILQ_INSERT_BEFORE(m2,
1077 m, m_pqe);
1078 }
1079 }
1080 return (0);
1081 }
1082 }
1083 TAILQ_INSERT_TAIL(q, m, m_qe);
1084 if (!is_pshared_mutex(m))
1085 TAILQ_INSERT_TAIL(qp, m, m_pqe);
1086 }
1087 }
1088 return (0);
1089 }
1090
1091 int
_pthread_mutex_getspinloops_np(pthread_mutex_t * mutex,int * count)1092 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
1093 {
1094 struct pthread_mutex *m;
1095 int ret;
1096
1097 ret = check_and_init_mutex(mutex, &m);
1098 if (ret == 0)
1099 *count = m->m_spinloops;
1100 return (ret);
1101 }
1102
1103 int
__pthread_mutex_setspinloops_np(pthread_mutex_t * mutex,int count)1104 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
1105 {
1106 struct pthread_mutex *m;
1107 int ret;
1108
1109 ret = check_and_init_mutex(mutex, &m);
1110 if (ret == 0)
1111 m->m_spinloops = count;
1112 return (ret);
1113 }
1114
1115 int
_pthread_mutex_getyieldloops_np(pthread_mutex_t * mutex,int * count)1116 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
1117 {
1118 struct pthread_mutex *m;
1119 int ret;
1120
1121 ret = check_and_init_mutex(mutex, &m);
1122 if (ret == 0)
1123 *count = m->m_yieldloops;
1124 return (ret);
1125 }
1126
1127 int
__pthread_mutex_setyieldloops_np(pthread_mutex_t * mutex,int count)1128 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
1129 {
1130 struct pthread_mutex *m;
1131 int ret;
1132
1133 ret = check_and_init_mutex(mutex, &m);
1134 if (ret == 0)
1135 m->m_yieldloops = count;
1136 return (0);
1137 }
1138
1139 int
_pthread_mutex_isowned_np(pthread_mutex_t * mutex)1140 _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
1141 {
1142 struct pthread_mutex *m;
1143
1144 if (*mutex == THR_PSHARED_PTR) {
1145 m = __thr_pshared_offpage(mutex, 0);
1146 if (m == NULL)
1147 return (0);
1148 shared_mutex_init(m, NULL);
1149 } else {
1150 m = *mutex;
1151 if (m <= THR_MUTEX_DESTROYED)
1152 return (0);
1153 }
1154 return (PMUTEX_OWNER_ID(m) == TID(_get_curthread()));
1155 }
1156
1157 int
_mutex_owned(struct pthread * curthread,const struct pthread_mutex * mp)1158 _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
1159 {
1160
1161 if (__predict_false(mp <= THR_MUTEX_DESTROYED)) {
1162 if (mp == THR_MUTEX_DESTROYED)
1163 return (EINVAL);
1164 return (EPERM);
1165 }
1166 if (PMUTEX_OWNER_ID(mp) != TID(curthread))
1167 return (EPERM);
1168 return (0);
1169 }
1170
1171 int
_pthread_mutex_consistent(pthread_mutex_t * mutex)1172 _pthread_mutex_consistent(pthread_mutex_t *mutex)
1173 {
1174 struct pthread_mutex *m;
1175 struct pthread *curthread;
1176
1177 if (*mutex == THR_PSHARED_PTR) {
1178 m = __thr_pshared_offpage(mutex, 0);
1179 if (m == NULL)
1180 return (EINVAL);
1181 shared_mutex_init(m, NULL);
1182 } else {
1183 m = *mutex;
1184 if (m <= THR_MUTEX_DESTROYED)
1185 return (EINVAL);
1186 }
1187 curthread = _get_curthread();
1188 if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) !=
1189 (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT))
1190 return (EINVAL);
1191 if (PMUTEX_OWNER_ID(m) != TID(curthread))
1192 return (EPERM);
1193 m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT;
1194 return (0);
1195 }
1196