Lines Matching refs:rw
97 #define rw_wowner(rw) \ argument
98 ((rw)->rw_lock & RW_LOCK_READ ? NULL : \
99 (struct thread *)RW_OWNER((rw)->rw_lock))
105 #define rw_recursed(rw) ((rw)->rw_recurse != 0) argument
110 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread) argument
117 #define rw_owner(rw) rw_wowner(rw) argument
120 #define _rw_assert(rw, what, file, line) argument
133 struct rwlock *rw; in lock_rw() local
135 rw = (struct rwlock *)lock; in lock_rw()
137 rw_wlock(rw); in lock_rw()
139 rw_rlock(rw); in lock_rw()
145 struct rwlock *rw; in unlock_rw() local
147 rw = (struct rwlock *)lock; in unlock_rw()
148 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED); in unlock_rw()
149 if (rw->rw_lock & RW_LOCK_READ) { in unlock_rw()
150 rw_runlock(rw); in unlock_rw()
153 rw_wunlock(rw); in unlock_rw()
162 struct rwlock *rw = (struct rwlock *)lock; in owner_rw() local
163 uintptr_t x = rw->rw_lock; in owner_rw()
165 *owner = rw_wowner(rw); in owner_rw()
172 rw_init_flags(struct rwlock *rw, const char *name, int opts) in rw_init_flags() argument
178 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock, in rw_init_flags()
180 &rw->rw_lock)); in rw_init_flags()
194 rw->rw_lock = RW_UNLOCKED; in rw_init_flags()
195 rw->rw_recurse = 0; in rw_init_flags()
196 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags); in rw_init_flags()
200 rw_destroy(struct rwlock *rw) in rw_destroy() argument
203 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw)); in rw_destroy()
204 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw)); in rw_destroy()
205 rw->rw_lock = RW_DESTROYED; in rw_destroy()
206 lock_destroy(&rw->lock_object); in rw_destroy()
226 rw_wowned(struct rwlock *rw) in rw_wowned() argument
229 return (rw_wowner(rw) == curthread); in rw_wowned()
233 _rw_wlock(struct rwlock *rw, const char *file, int line) in _rw_wlock() argument
240 curthread, rw->lock_object.lo_name, file, line)); in _rw_wlock()
241 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_wlock()
243 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, in _rw_wlock()
245 __rw_wlock(rw, curthread, file, line); in _rw_wlock()
246 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); in _rw_wlock()
247 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); in _rw_wlock()
252 _rw_try_wlock(struct rwlock *rw, const char *file, int line) in _rw_try_wlock() argument
261 curthread, rw->lock_object.lo_name, file, line)); in _rw_try_wlock()
262 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_try_wlock()
265 if (rw_wlocked(rw) && in _rw_try_wlock()
266 (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { in _rw_try_wlock()
267 rw->rw_recurse++; in _rw_try_wlock()
270 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, in _rw_try_wlock()
273 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); in _rw_try_wlock()
275 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, in _rw_try_wlock()
277 if (!rw_recursed(rw)) in _rw_try_wlock()
279 rw, 0, 0, file, line); in _rw_try_wlock()
286 _rw_wunlock(struct rwlock *rw, const char *file, int line) in _rw_wunlock() argument
291 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_wunlock()
293 _rw_assert(rw, RA_WLOCKED, file, line); in _rw_wunlock()
295 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); in _rw_wunlock()
296 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, in _rw_wunlock()
298 if (!rw_recursed(rw)) in _rw_wunlock()
299 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_WUNLOCK_RELEASE, rw); in _rw_wunlock()
300 __rw_wunlock(rw, curthread, file, line); in _rw_wunlock()
315 _rw_rlock(struct rwlock *rw, const char *file, int line) in _rw_rlock() argument
341 curthread, rw->lock_object.lo_name, file, line)); in _rw_rlock()
342 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_rlock()
344 KASSERT(rw_wowner(rw) != curthread, in _rw_rlock()
346 rw->lock_object.lo_name, file, line)); in _rw_rlock()
347 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); in _rw_rlock()
350 all_time -= lockstat_nsecs(&rw->lock_object); in _rw_rlock()
351 state = rw->rw_lock; in _rw_rlock()
367 v = rw->rw_lock; in _rw_rlock()
374 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, in _rw_rlock()
376 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_rlock()
379 rw, (void *)v, in _rw_rlock()
388 lock_profile_obtain_lock_failed(&rw->lock_object, in _rw_rlock()
400 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_rlock()
403 __func__, rw, owner); in _rw_rlock()
404 while ((struct thread*)RW_OWNER(rw->rw_lock) == in _rw_rlock()
416 v = rw->rw_lock; in _rw_rlock()
432 ts = turnstile_trywait(&rw->lock_object); in _rw_rlock()
438 v = rw->rw_lock; in _rw_rlock()
473 if (!atomic_cmpset_ptr(&rw->rw_lock, v, in _rw_rlock()
478 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_rlock()
480 __func__, rw); in _rw_rlock()
487 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_rlock()
489 rw); in _rw_rlock()
491 sleep_time -= lockstat_nsecs(&rw->lock_object); in _rw_rlock()
493 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); in _rw_rlock()
495 sleep_time += lockstat_nsecs(&rw->lock_object); in _rw_rlock()
498 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_rlock()
500 __func__, rw); in _rw_rlock()
503 all_time += lockstat_nsecs(&rw->lock_object); in _rw_rlock()
505 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, in _rw_rlock()
511 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time, in _rw_rlock()
520 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, in _rw_rlock()
522 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); in _rw_rlock()
523 WITNESS_LOCK(&rw->lock_object, 0, file, line); in _rw_rlock()
529 _rw_try_rlock(struct rwlock *rw, const char *file, int line) in _rw_try_rlock() argument
538 curthread, rw->lock_object.lo_name, file, line)); in _rw_try_rlock()
541 x = rw->rw_lock; in _rw_try_rlock()
542 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_try_rlock()
546 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { in _rw_try_rlock()
547 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, in _rw_try_rlock()
549 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); in _rw_try_rlock()
551 rw, 0, 0, file, line); in _rw_try_rlock()
558 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); in _rw_try_rlock()
563 _rw_runlock(struct rwlock *rw, const char *file, int line) in _rw_runlock() argument
571 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_runlock()
573 _rw_assert(rw, RA_RLOCKED, file, line); in _rw_runlock()
576 WITNESS_UNLOCK(&rw->lock_object, 0, file, line); in _rw_runlock()
577 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line); in _rw_runlock()
586 x = rw->rw_lock; in _rw_runlock()
588 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, in _rw_runlock()
590 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_runlock()
593 __func__, rw, (void *)x, in _rw_runlock()
606 if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, in _rw_runlock()
608 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_runlock()
610 __func__, rw); in _rw_runlock()
619 turnstile_chain_lock(&rw->lock_object); in _rw_runlock()
620 v = rw->rw_lock & (RW_LOCK_WAITERS | RW_LOCK_WRITE_SPINNER); in _rw_runlock()
645 if (!atomic_cmpset_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, in _rw_runlock()
647 turnstile_chain_unlock(&rw->lock_object); in _rw_runlock()
650 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_runlock()
652 __func__, rw); in _rw_runlock()
661 ts = turnstile_lookup(&rw->lock_object); in _rw_runlock()
665 turnstile_chain_unlock(&rw->lock_object); in _rw_runlock()
668 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw); in _rw_runlock()
677 _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) in _rw_wlock_hard() argument
701 if (rw_wlocked(rw)) { in _rw_wlock_hard()
702 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, in _rw_wlock_hard()
704 __func__, rw->lock_object.lo_name, file, line)); in _rw_wlock_hard()
705 rw->rw_recurse++; in _rw_wlock_hard()
706 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
707 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); in _rw_wlock_hard()
711 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
713 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); in _rw_wlock_hard()
716 all_time -= lockstat_nsecs(&rw->lock_object); in _rw_wlock_hard()
717 state = rw->rw_lock; in _rw_wlock_hard()
719 while (!_rw_write_lock(rw, tid)) { in _rw_wlock_hard()
726 lock_profile_obtain_lock_failed(&rw->lock_object, in _rw_wlock_hard()
734 v = rw->rw_lock; in _rw_wlock_hard()
737 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
739 __func__, rw, owner); in _rw_wlock_hard()
740 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && in _rw_wlock_hard()
752 if (!atomic_cmpset_ptr(&rw->rw_lock, v, in _rw_wlock_hard()
759 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0) in _rw_wlock_hard()
770 ts = turnstile_trywait(&rw->lock_object); in _rw_wlock_hard()
771 v = rw->rw_lock; in _rw_wlock_hard()
799 if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, tid | x)) { in _rw_wlock_hard()
815 if (!atomic_cmpset_ptr(&rw->rw_lock, v, in _rw_wlock_hard()
820 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
822 __func__, rw); in _rw_wlock_hard()
828 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
830 rw); in _rw_wlock_hard()
832 sleep_time -= lockstat_nsecs(&rw->lock_object); in _rw_wlock_hard()
834 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); in _rw_wlock_hard()
836 sleep_time += lockstat_nsecs(&rw->lock_object); in _rw_wlock_hard()
839 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wlock_hard()
841 __func__, rw); in _rw_wlock_hard()
847 all_time += lockstat_nsecs(&rw->lock_object); in _rw_wlock_hard()
849 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, in _rw_wlock_hard()
855 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time, in _rw_wlock_hard()
859 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, in _rw_wlock_hard()
869 _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) in _rw_wunlock_hard() argument
878 if (rw_wlocked(rw) && rw_recursed(rw)) { in _rw_wunlock_hard()
879 rw->rw_recurse--; in _rw_wunlock_hard()
880 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wunlock_hard()
881 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); in _rw_wunlock_hard()
885 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), in _rw_wunlock_hard()
888 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wunlock_hard()
889 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); in _rw_wunlock_hard()
891 turnstile_chain_lock(&rw->lock_object); in _rw_wunlock_hard()
892 ts = turnstile_lookup(&rw->lock_object); in _rw_wunlock_hard()
912 if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { in _rw_wunlock_hard()
914 v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); in _rw_wunlock_hard()
919 if (LOCK_LOG_TEST(&rw->lock_object, 0)) in _rw_wunlock_hard()
920 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, in _rw_wunlock_hard()
923 atomic_store_rel_ptr(&rw->rw_lock, v); in _rw_wunlock_hard()
925 turnstile_chain_unlock(&rw->lock_object); in _rw_wunlock_hard()
934 _rw_try_upgrade(struct rwlock *rw, const char *file, int line) in _rw_try_upgrade() argument
943 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_try_upgrade()
945 _rw_assert(rw, RA_RLOCKED, file, line); in _rw_try_upgrade()
957 v = rw->rw_lock; in _rw_try_upgrade()
961 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid); in _rw_try_upgrade()
970 ts = turnstile_trywait(&rw->lock_object); in _rw_try_upgrade()
971 v = rw->rw_lock; in _rw_try_upgrade()
982 x = rw->rw_lock & RW_LOCK_WAITERS; in _rw_try_upgrade()
983 success = atomic_cmpset_ptr(&rw->rw_lock, v, tid | x); in _rw_try_upgrade()
993 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); in _rw_try_upgrade()
996 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, in _rw_try_upgrade()
998 LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw); in _rw_try_upgrade()
1007 _rw_downgrade(struct rwlock *rw, const char *file, int line) in _rw_downgrade() argument
1016 KASSERT(rw->rw_lock != RW_DESTROYED, in _rw_downgrade()
1018 _rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line); in _rw_downgrade()
1020 if (rw_recursed(rw)) in _rw_downgrade()
1024 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); in _rw_downgrade()
1032 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) in _rw_downgrade()
1039 turnstile_chain_lock(&rw->lock_object); in _rw_downgrade()
1040 v = rw->rw_lock & RW_LOCK_WAITERS; in _rw_downgrade()
1049 ts = turnstile_lookup(&rw->lock_object); in _rw_downgrade()
1053 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); in _rw_downgrade()
1063 turnstile_chain_unlock(&rw->lock_object); in _rw_downgrade()
1066 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); in _rw_downgrade()
1067 LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw); in _rw_downgrade()
1081 _rw_assert(struct rwlock *rw, int what, const char *file, int line) in _rw_assert() argument
1094 witness_assert(&rw->lock_object, what, file, line); in _rw_assert()
1101 if (rw->rw_lock == RW_UNLOCKED || in _rw_assert()
1102 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED || in _rw_assert()
1103 rw_wowner(rw) != curthread))) in _rw_assert()
1105 rw->lock_object.lo_name, (what & RA_RLOCKED) ? in _rw_assert()
1108 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) { in _rw_assert()
1109 if (rw_recursed(rw)) { in _rw_assert()
1112 rw->lock_object.lo_name, file, in _rw_assert()
1116 rw->lock_object.lo_name, file, line); in _rw_assert()
1123 if (rw_wowner(rw) != curthread) in _rw_assert()
1125 rw->lock_object.lo_name, file, line); in _rw_assert()
1126 if (rw_recursed(rw)) { in _rw_assert()
1129 rw->lock_object.lo_name, file, line); in _rw_assert()
1132 rw->lock_object.lo_name, file, line); in _rw_assert()
1136 witness_assert(&rw->lock_object, what, file, line); in _rw_assert()
1142 if (rw_wowner(rw) == curthread) in _rw_assert()
1144 rw->lock_object.lo_name, file, line); in _rw_assert()
1158 struct rwlock *rw; in db_show_rwlock() local
1161 rw = (struct rwlock *)lock; in db_show_rwlock()
1164 if (rw->rw_lock == RW_UNLOCKED) in db_show_rwlock()
1166 else if (rw->rw_lock == RW_DESTROYED) { in db_show_rwlock()
1169 } else if (rw->rw_lock & RW_LOCK_READ) in db_show_rwlock()
1171 (uintmax_t)(RW_READERS(rw->rw_lock))); in db_show_rwlock()
1173 td = rw_wowner(rw); in db_show_rwlock()
1176 if (rw_recursed(rw)) in db_show_rwlock()
1177 db_printf(" recursed: %u\n", rw->rw_recurse); in db_show_rwlock()
1180 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { in db_show_rwlock()