81 static int	mutex_qidx(struct pthread_mutex *m);
82 static bool	is_robust_mutex(struct pthread_mutex *m);
83 static bool	is_pshared_mutex(struct pthread_mutex *m);
120 mutex_init_link(struct pthread_mutex *m __unused)  in mutex_init_link()
124 	m->m_qe.tqe_prev = NULL;  in mutex_init_link()
125 	m->m_qe.tqe_next = NULL;  in mutex_init_link()
126 	m->m_pqe.tqe_prev = NULL;  in mutex_init_link()
127 	m->m_pqe.tqe_next = NULL;  in mutex_init_link()
132 mutex_assert_is_owned(struct pthread_mutex *m __unused)  in mutex_assert_is_owned()
136 	if (__predict_false(m->m_qe.tqe_prev == NULL))  in mutex_assert_is_owned()
138 		    m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);  in mutex_assert_is_owned()
144     struct pthread_mutex *m __unused)  in mutex_assert_not_owned()
148 	if (__predict_false(m->m_qe.tqe_prev != NULL ||  in mutex_assert_not_owned()
149 	    m->m_qe.tqe_next != NULL))  in mutex_assert_not_owned()
151 		    m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);  in mutex_assert_not_owned()
152 	if (__predict_false(is_robust_mutex(m) &&  in mutex_assert_not_owned()
153 	    (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||  in mutex_assert_not_owned()
154 	    (is_pshared_mutex(m) && curthread->robust_list ==  in mutex_assert_not_owned()
155 	    (uintptr_t)&m->m_lock) ||  in mutex_assert_not_owned()
156 	    (!is_pshared_mutex(m) && curthread->priv_robust_list ==  in mutex_assert_not_owned()
157 	    (uintptr_t)&m->m_lock))))  in mutex_assert_not_owned()
160 		    m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,  in mutex_assert_not_owned()
161 		    m->m_rb_prev, (void *)curthread->robust_list,  in mutex_assert_not_owned()
167 is_pshared_mutex(struct pthread_mutex *m)  in is_pshared_mutex()  argument
170 	return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);  in is_pshared_mutex()
174 is_robust_mutex(struct pthread_mutex *m)  in is_robust_mutex()  argument
177 	return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0);  in is_robust_mutex()
181 _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)  in _mutex_enter_robust()  argument
188 	if (!is_robust_mutex(m))  in _mutex_enter_robust()
192 	curthread->inact_mtx = (uintptr_t)&m->m_lock;  in _mutex_enter_robust()
197 _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)  in _mutex_leave_robust()
201 	if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))  in _mutex_leave_robust()
320 set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)  in set_inherited_priority()  argument
324 	m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);  in set_inherited_priority()
326 		m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];  in set_inherited_priority()
328 		m->m_lock.m_ceilings[1] = -1;  in set_inherited_priority()
439 	struct pthread_mutex *m;  in queue_fork()  local
442 	TAILQ_FOREACH(m, qp, m_pqe) {  in queue_fork()
443 		TAILQ_INSERT_TAIL(q, m, m_qe);  in queue_fork()
444 		m->m_lock.m_owner = TID(curthread) | bit;  in queue_fork()
464 	pthread_mutex_t m, m1;  in _thr_mutex_destroy()  local
467 	m = *mutex;  in _thr_mutex_destroy()
468 	if (m < THR_MUTEX_DESTROYED) {  in _thr_mutex_destroy()
470 	} else if (m == THR_MUTEX_DESTROYED) {  in _thr_mutex_destroy()
473 		if (m == THR_PSHARED_PTR) {  in _thr_mutex_destroy()
486 		if (PMUTEX_OWNER_ID(m) != 0 &&  in _thr_mutex_destroy()
487 		    (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) {  in _thr_mutex_destroy()
491 			mutex_assert_not_owned(_get_curthread(), m);  in _thr_mutex_destroy()
492 			__thr_free(m);  in _thr_mutex_destroy()
501 mutex_qidx(struct pthread_mutex *m)  in mutex_qidx()  argument
504 	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)  in mutex_qidx()
506 	return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP);  in mutex_qidx()
522 enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,  in enqueue_mutex()  argument
531 		mutex_assert_not_owned(curthread, m);  in enqueue_mutex()
532 	qidx = mutex_qidx(m);  in enqueue_mutex()
533 	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);  in enqueue_mutex()
534 	if (!is_pshared_mutex(m))  in enqueue_mutex()
535 		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);  in enqueue_mutex()
536 	if (is_robust_mutex(m)) {  in enqueue_mutex()
537 		rl = is_pshared_mutex(m) ? &curthread->robust_list :  in enqueue_mutex()
539 		m->m_rb_prev = NULL;  in enqueue_mutex()
543 			m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock;  in enqueue_mutex()
544 			m1->m_rb_prev = m;  in enqueue_mutex()
547 			m->m_lock.m_rb_lnk = 0;  in enqueue_mutex()
549 		*rl = (uintptr_t)&m->m_lock;  in enqueue_mutex()
554 dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)  in dequeue_mutex()  argument
559 	mutex_assert_is_owned(m);  in dequeue_mutex()
560 	qidx = mutex_qidx(m);  in dequeue_mutex()
561 	if (is_robust_mutex(m)) {  in dequeue_mutex()
562 		mp = m->m_rb_prev;  in dequeue_mutex()
564 			if (is_pshared_mutex(m)) {  in dequeue_mutex()
565 				curthread->robust_list = m->m_lock.m_rb_lnk;  in dequeue_mutex()
568 				    m->m_lock.m_rb_lnk;  in dequeue_mutex()
571 			mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk;  in dequeue_mutex()
573 		if (m->m_lock.m_rb_lnk != 0) {  in dequeue_mutex()
574 			mn = __containerof((void *)m->m_lock.m_rb_lnk,  in dequeue_mutex()
576 			mn->m_rb_prev = m->m_rb_prev;  in dequeue_mutex()
578 		m->m_lock.m_rb_lnk = 0;  in dequeue_mutex()
579 		m->m_rb_prev = NULL;  in dequeue_mutex()
581 	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);  in dequeue_mutex()
582 	if (!is_pshared_mutex(m))  in dequeue_mutex()
583 		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);  in dequeue_mutex()
584 	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)  in dequeue_mutex()
585 		set_inherited_priority(curthread, m);  in dequeue_mutex()
586 	mutex_init_link(m);  in dequeue_mutex()
590 check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)  in check_and_init_mutex()  argument
594 	*m = *mutex;  in check_and_init_mutex()
596 	if (__predict_false(*m == THR_PSHARED_PTR)) {  in check_and_init_mutex()
597 		*m = __thr_pshared_offpage(mutex, 0);  in check_and_init_mutex()
598 		if (*m == NULL)  in check_and_init_mutex()
601 			shared_mutex_init(*m, NULL);  in check_and_init_mutex()
602 	} else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {  in check_and_init_mutex()
603 		if (*m == THR_MUTEX_DESTROYED) {  in check_and_init_mutex()
608 				*m = *mutex;  in check_and_init_mutex()
618 	struct pthread_mutex *m;  in __Tthr_mutex_trylock()  local
622 	ret = check_and_init_mutex(mutex, &m);  in __Tthr_mutex_trylock()
627 	if (m->m_flags & PMUTEX_FLAG_PRIVATE)  in __Tthr_mutex_trylock()
629 	robust = _mutex_enter_robust(curthread, m);  in __Tthr_mutex_trylock()
630 	ret = _thr_umutex_trylock(&m->m_lock, id);  in __Tthr_mutex_trylock()
632 		enqueue_mutex(curthread, m, ret);  in __Tthr_mutex_trylock()
634 			m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;  in __Tthr_mutex_trylock()
635 	} else if (PMUTEX_OWNER_ID(m) == id) {  in __Tthr_mutex_trylock()
636 		ret = mutex_self_trylock(m);  in __Tthr_mutex_trylock()
639 		_mutex_leave_robust(curthread, m);  in __Tthr_mutex_trylock()
641 	    (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0)  in __Tthr_mutex_trylock()
647 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,  in mutex_lock_sleep()  argument
654 	if (PMUTEX_OWNER_ID(m) == id)  in mutex_lock_sleep()
655 		return (mutex_self_lock(m, abstime));  in mutex_lock_sleep()
663 	if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT |  in mutex_lock_sleep()
670 	count = m->m_spinloops;  in mutex_lock_sleep()
672 		owner = m->m_lock.m_owner;  in mutex_lock_sleep()
674 			if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,  in mutex_lock_sleep()
684 	count = m->m_yieldloops;  in mutex_lock_sleep()
687 		owner = m->m_lock.m_owner;  in mutex_lock_sleep()
689 			if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,  in mutex_lock_sleep()
699 		ret = __thr_umutex_lock(&m->m_lock, id);  in mutex_lock_sleep()
704 		ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);  in mutex_lock_sleep()
707 		enqueue_mutex(curthread, m, ret);  in mutex_lock_sleep()
709 			m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;  in mutex_lock_sleep()
715 mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime,  in mutex_lock_common()  argument
723 	if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)  in mutex_lock_common()
726 		robust = _mutex_enter_robust(curthread, m);  in mutex_lock_common()
727 	ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));  in mutex_lock_common()
729 		enqueue_mutex(curthread, m, ret);  in mutex_lock_common()
731 			m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;  in mutex_lock_common()
733 		ret = mutex_lock_sleep(curthread, m, abstime);  in mutex_lock_common()
736 		_mutex_leave_robust(curthread, m);  in mutex_lock_common()
738 	    (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach)  in mutex_lock_common()
746 	struct pthread_mutex *m;  in __Tthr_mutex_lock()  local
750 	ret = check_and_init_mutex(mutex, &m);  in __Tthr_mutex_lock()
752 		ret = mutex_lock_common(m, NULL, false, false);  in __Tthr_mutex_lock()
760 	struct pthread_mutex *m;  in __pthread_mutex_timedlock()  local
764 	ret = check_and_init_mutex(mutex, &m);  in __pthread_mutex_timedlock()
766 		ret = mutex_lock_common(m, abstime, false, false);  in __pthread_mutex_timedlock()
787 _mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist)  in _mutex_cv_lock()  argument
791 	error = mutex_lock_common(m, NULL, true, rb_onlist);  in _mutex_cv_lock()
793 		m->m_count = count;  in _mutex_cv_lock()
798 _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer)  in _mutex_cv_unlock()  argument
804 	*count = m->m_count;  in _mutex_cv_unlock()
805 	m->m_count = 0;  in _mutex_cv_unlock()
806 	(void)mutex_unlock_common(m, true, defer);  in _mutex_cv_unlock()
811 _mutex_cv_attach(struct pthread_mutex *m, int count)  in _mutex_cv_attach()  argument
816 	enqueue_mutex(curthread, m, 0);  in _mutex_cv_attach()
817 	m->m_count = count;  in _mutex_cv_attach()
854 mutex_self_trylock(struct pthread_mutex *m)  in mutex_self_trylock()  argument
858 	switch (PMUTEX_TYPE(m->m_flags)) {  in mutex_self_trylock()
867 		if (m->m_count + 1 > 0) {  in mutex_self_trylock()
868 			m->m_count++;  in mutex_self_trylock()
883 mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)  in mutex_self_lock()  argument
888 	switch (PMUTEX_TYPE(m->m_flags)) {  in mutex_self_lock()
936 		if (m->m_count + 1 > 0) {  in mutex_self_lock()
937 			m->m_count++;  in mutex_self_lock()
952 mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer)  in mutex_unlock_common()  argument
958 	if (__predict_false(m <= THR_MUTEX_DESTROYED)) {  in mutex_unlock_common()
959 		if (m == THR_MUTEX_DESTROYED)  in mutex_unlock_common()
970 	if (__predict_false(PMUTEX_OWNER_ID(m) != id))  in mutex_unlock_common()
974 	private = (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0;  in mutex_unlock_common()
975 	if (__predict_false(PMUTEX_TYPE(m->m_flags) ==  in mutex_unlock_common()
976 	    PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) {  in mutex_unlock_common()
977 		m->m_count--;  in mutex_unlock_common()
979 		if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {  in mutex_unlock_common()
981 			m->m_flags &= ~PMUTEX_FLAG_DEFERRED;  in mutex_unlock_common()
985 		robust = _mutex_enter_robust(curthread, m);  in mutex_unlock_common()
986 		dequeue_mutex(curthread, m);  in mutex_unlock_common()
987 		error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);  in mutex_unlock_common()
997 			_mutex_leave_robust(curthread, m);  in mutex_unlock_common()
1008 	struct pthread_mutex *m;  in _pthread_mutex_getprioceiling()  local
1011 		m = __thr_pshared_offpage(__DECONST(void *, mutex), 0);  in _pthread_mutex_getprioceiling()
1012 		if (m == NULL)  in _pthread_mutex_getprioceiling()
1014 		shared_mutex_init(m, NULL);  in _pthread_mutex_getprioceiling()
1016 		m = *mutex;  in _pthread_mutex_getprioceiling()
1017 		if (m <= THR_MUTEX_DESTROYED)  in _pthread_mutex_getprioceiling()
1020 	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)  in _pthread_mutex_getprioceiling()
1022 	*prioceiling = m->m_lock.m_ceilings[0];  in _pthread_mutex_getprioceiling()
1031 	struct pthread_mutex *m, *m1, *m2;  in _pthread_mutex_setprioceiling()  local
1036 		m = __thr_pshared_offpage(mutex, 0);  in _pthread_mutex_setprioceiling()
1037 		if (m == NULL)  in _pthread_mutex_setprioceiling()
1039 		shared_mutex_init(m, NULL);  in _pthread_mutex_setprioceiling()
1041 		m = *mutex;  in _pthread_mutex_setprioceiling()
1042 		if (m <= THR_MUTEX_DESTROYED)  in _pthread_mutex_setprioceiling()
1045 	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)  in _pthread_mutex_setprioceiling()
1048 	ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);  in _pthread_mutex_setprioceiling()
1053 	if (PMUTEX_OWNER_ID(m) == TID(curthread)) {  in _pthread_mutex_setprioceiling()
1054 		mutex_assert_is_owned(m);  in _pthread_mutex_setprioceiling()
1055 		m1 = TAILQ_PREV(m, mutex_queue, m_qe);  in _pthread_mutex_setprioceiling()
1056 		m2 = TAILQ_NEXT(m, m_qe);  in _pthread_mutex_setprioceiling()
1059 			qidx = mutex_qidx(m);  in _pthread_mutex_setprioceiling()
1062 			TAILQ_REMOVE(q, m, m_qe);  in _pthread_mutex_setprioceiling()
1063 			if (!is_pshared_mutex(m))  in _pthread_mutex_setprioceiling()
1064 				TAILQ_REMOVE(qp, m, m_pqe);  in _pthread_mutex_setprioceiling()
1067 					TAILQ_INSERT_BEFORE(m2, m, m_qe);  in _pthread_mutex_setprioceiling()
1068 					if (!is_pshared_mutex(m)) {  in _pthread_mutex_setprioceiling()
1076 							    m, m_pqe);  in _pthread_mutex_setprioceiling()
1079 							    m, m_pqe);  in _pthread_mutex_setprioceiling()
1085 			TAILQ_INSERT_TAIL(q, m, m_qe);  in _pthread_mutex_setprioceiling()
1086 			if (!is_pshared_mutex(m))  in _pthread_mutex_setprioceiling()
1087 				TAILQ_INSERT_TAIL(qp, m, m_pqe);  in _pthread_mutex_setprioceiling()
1096 	struct pthread_mutex *m;  in _pthread_mutex_getspinloops_np()  local
1099 	ret = check_and_init_mutex(mutex, &m);  in _pthread_mutex_getspinloops_np()
1101 		*count = m->m_spinloops;  in _pthread_mutex_getspinloops_np()
1108 	struct pthread_mutex *m;  in __pthread_mutex_setspinloops_np()  local
1111 	ret = check_and_init_mutex(mutex, &m);  in __pthread_mutex_setspinloops_np()
1113 		m->m_spinloops = count;  in __pthread_mutex_setspinloops_np()
1120 	struct pthread_mutex *m;  in _pthread_mutex_getyieldloops_np()  local
1123 	ret = check_and_init_mutex(mutex, &m);  in _pthread_mutex_getyieldloops_np()
1125 		*count = m->m_yieldloops;  in _pthread_mutex_getyieldloops_np()
1132 	struct pthread_mutex *m;  in __pthread_mutex_setyieldloops_np()  local
1135 	ret = check_and_init_mutex(mutex, &m);  in __pthread_mutex_setyieldloops_np()
1137 		m->m_yieldloops = count;  in __pthread_mutex_setyieldloops_np()
1144 	struct pthread_mutex *m;  in _pthread_mutex_isowned_np()  local
1147 		m = __thr_pshared_offpage(mutex, 0);  in _pthread_mutex_isowned_np()
1148 		if (m == NULL)  in _pthread_mutex_isowned_np()
1150 		shared_mutex_init(m, NULL);  in _pthread_mutex_isowned_np()
1152 		m = *mutex;  in _pthread_mutex_isowned_np()
1153 		if (m <= THR_MUTEX_DESTROYED)  in _pthread_mutex_isowned_np()
1156 	return (PMUTEX_OWNER_ID(m) == TID(_get_curthread()));  in _pthread_mutex_isowned_np()
1176 	struct pthread_mutex *m;  in _Tthr_mutex_consistent()  local
1180 		m = __thr_pshared_offpage(mutex, 0);  in _Tthr_mutex_consistent()
1181 		if (m == NULL)  in _Tthr_mutex_consistent()
1183 		shared_mutex_init(m, NULL);  in _Tthr_mutex_consistent()
1185 		m = *mutex;  in _Tthr_mutex_consistent()
1186 		if (m <= THR_MUTEX_DESTROYED)  in _Tthr_mutex_consistent()
1190 	if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) !=  in _Tthr_mutex_consistent()
1193 	if (PMUTEX_OWNER_ID(m) != TID(curthread))  in _Tthr_mutex_consistent()
1195 	m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT;  in _Tthr_mutex_consistent()