1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include "opt_ktrace.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/limits.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/proc.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/condvar.h>
41 #include <sys/sched.h>
42 #include <sys/signalvar.h>
43 #include <sys/sleepqueue.h>
44 #include <sys/resourcevar.h>
45 #ifdef KTRACE
46 #include <sys/ktrace.h>
47 #include <sys/uio.h>
48 #include <sys/user.h>
49 #endif
50
51 /*
52 * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound,
53 * cv_signal must manually check the wait queue for threads.
54 */
55 #define CV_WAITERS_BOUND INT_MAX
56
57 #define CV_WAITERS_INC(cvp) do { \
58 if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \
59 (cvp)->cv_waiters++; \
60 } while (0)
61
62 /*
63 * Common sanity checks for cv_wait* functions.
64 */
65 #define CV_ASSERT(cvp, lock, td) do { \
66 KASSERT((td) != NULL, ("%s: td NULL", __func__)); \
67 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \
68 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \
69 KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \
70 } while (0)
71
72 /*
73 * Initialize a condition variable. Must be called before use.
74 */
75 void
cv_init(struct cv * cvp,const char * desc)76 cv_init(struct cv *cvp, const char *desc)
77 {
78
79 cvp->cv_description = desc;
80 cvp->cv_waiters = 0;
81 }
82
83 /*
84 * Destroy a condition variable. The condition variable must be re-initialized
85 * in order to be re-used.
86 */
87 void
cv_destroy(struct cv * cvp)88 cv_destroy(struct cv *cvp)
89 {
90 #ifdef INVARIANTS
91 struct sleepqueue *sq;
92
93 sleepq_lock(cvp);
94 sq = sleepq_lookup(cvp);
95 sleepq_release(cvp);
96 KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
97 #endif
98 }
99
100 /*
101 * Wait on a condition variable. The current thread is placed on the condition
102 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same
103 * condition variable will resume the thread. The mutex is released before
104 * sleeping and will be held on return. It is recommended that the mutex be
105 * held when cv_signal or cv_broadcast are called.
106 */
107 void
_cv_wait(struct cv * cvp,struct lock_object * lock)108 _cv_wait(struct cv *cvp, struct lock_object *lock)
109 {
110 WITNESS_SAVE_DECL(lock_witness);
111 #ifdef KTRACE
112 char wmesg[WMESGLEN + 1];
113 #endif
114 struct lock_class *class;
115 struct thread *td;
116 uintptr_t lock_state;
117
118 td = curthread;
119 CV_ASSERT(cvp, lock, td);
120 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
121 "Waiting on \"%s\"", cvp->cv_description);
122
123 if (SCHEDULER_STOPPED_TD(td))
124 return;
125
126 #ifdef KTRACE
127 if (KTRPOINT(td, KTR_CSW)) {
128 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
129 ktrcsw(1, 0, wmesg);
130 } else {
131 wmesg[0] = '\0';
132 }
133 #endif
134
135 class = LOCK_CLASS(lock);
136 lock_state = 0;
137 sleepq_lock(cvp);
138
139 CV_WAITERS_INC(cvp);
140 if (lock == &Giant.lock_object)
141 mtx_assert(&Giant, MA_OWNED);
142 DROP_GIANT();
143
144 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
145 if (lock != &Giant.lock_object) {
146 if (class->lc_flags & LC_SLEEPABLE)
147 sleepq_release(cvp);
148 WITNESS_SAVE(lock, lock_witness);
149 lock_state = class->lc_unlock(lock);
150 if (class->lc_flags & LC_SLEEPABLE)
151 sleepq_lock(cvp);
152 }
153 sleepq_wait(cvp, 0);
154
155 #ifdef KTRACE
156 if (KTRPOINT(td, KTR_CSW))
157 ktrcsw(0, 0, wmesg);
158 #endif
159 PICKUP_GIANT();
160 if (lock != &Giant.lock_object) {
161 class->lc_lock(lock, lock_state);
162 WITNESS_RESTORE(lock, lock_witness);
163 }
164 }
165
166 /*
167 * Wait on a condition variable. This function differs from cv_wait by
168 * not acquiring the mutex after condition variable was signaled.
169 */
170 void
_cv_wait_unlock(struct cv * cvp,struct lock_object * lock)171 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
172 {
173 #ifdef KTRACE
174 char wmesg[WMESGLEN + 1];
175 #endif
176 struct lock_class *class;
177 struct thread *td;
178
179 td = curthread;
180 CV_ASSERT(cvp, lock, td);
181 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
182 "Waiting on \"%s\"", cvp->cv_description);
183 KASSERT(lock != &Giant.lock_object,
184 ("cv_wait_unlock cannot be used with Giant"));
185 class = LOCK_CLASS(lock);
186
187 if (SCHEDULER_STOPPED_TD(td)) {
188 class->lc_unlock(lock);
189 return;
190 }
191
192 #ifdef KTRACE
193 if (KTRPOINT(td, KTR_CSW)) {
194 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
195 ktrcsw(1, 0, wmesg);
196 } else {
197 wmesg[0] = '\0';
198 }
199 #endif
200
201 sleepq_lock(cvp);
202
203 CV_WAITERS_INC(cvp);
204 DROP_GIANT();
205
206 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
207 if (class->lc_flags & LC_SLEEPABLE)
208 sleepq_release(cvp);
209 class->lc_unlock(lock);
210 if (class->lc_flags & LC_SLEEPABLE)
211 sleepq_lock(cvp);
212 sleepq_wait(cvp, 0);
213
214 #ifdef KTRACE
215 if (KTRPOINT(td, KTR_CSW))
216 ktrcsw(0, 0, wmesg);
217 #endif
218 PICKUP_GIANT();
219 }
220
221 /*
222 * Wait on a condition variable, allowing interruption by signals. Return 0 if
223 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
224 * a signal was caught. If ERESTART is returned the system call should be
225 * restarted if possible.
226 */
227 int
_cv_wait_sig(struct cv * cvp,struct lock_object * lock)228 _cv_wait_sig(struct cv *cvp, struct lock_object *lock)
229 {
230 WITNESS_SAVE_DECL(lock_witness);
231 #ifdef KTRACE
232 char wmesg[WMESGLEN + 1];
233 #endif
234 struct lock_class *class;
235 struct thread *td;
236 uintptr_t lock_state;
237 int rval;
238
239 td = curthread;
240 CV_ASSERT(cvp, lock, td);
241 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
242 "Waiting on \"%s\"", cvp->cv_description);
243
244 if (SCHEDULER_STOPPED_TD(td))
245 return (0);
246
247 #ifdef KTRACE
248 if (KTRPOINT(td, KTR_CSW)) {
249 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
250 ktrcsw(1, 0, wmesg);
251 } else {
252 wmesg[0] = '\0';
253 }
254 #endif
255
256 class = LOCK_CLASS(lock);
257 lock_state = 0;
258 sleepq_lock(cvp);
259
260 CV_WAITERS_INC(cvp);
261 if (lock == &Giant.lock_object)
262 mtx_assert(&Giant, MA_OWNED);
263 DROP_GIANT();
264
265 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
266 SLEEPQ_INTERRUPTIBLE, 0);
267 if (lock != &Giant.lock_object) {
268 if (class->lc_flags & LC_SLEEPABLE)
269 sleepq_release(cvp);
270 WITNESS_SAVE(lock, lock_witness);
271 lock_state = class->lc_unlock(lock);
272 if (class->lc_flags & LC_SLEEPABLE)
273 sleepq_lock(cvp);
274 }
275 rval = sleepq_wait_sig(cvp, 0);
276
277 #ifdef KTRACE
278 if (KTRPOINT(td, KTR_CSW))
279 ktrcsw(0, 0, wmesg);
280 #endif
281 PICKUP_GIANT();
282 if (lock != &Giant.lock_object) {
283 class->lc_lock(lock, lock_state);
284 WITNESS_RESTORE(lock, lock_witness);
285 }
286
287 return (rval);
288 }
289
290 /*
291 * Wait on a condition variable for (at most) the value specified in sbt
292 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
293 * EWOULDBLOCK if the timeout expires.
294 */
295 int
_cv_timedwait_sbt(struct cv * cvp,struct lock_object * lock,sbintime_t sbt,sbintime_t pr,int flags)296 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
297 sbintime_t pr, int flags)
298 {
299 WITNESS_SAVE_DECL(lock_witness);
300 #ifdef KTRACE
301 char wmesg[WMESGLEN + 1];
302 #endif
303 struct lock_class *class;
304 struct thread *td;
305 int lock_state, rval;
306
307 td = curthread;
308 CV_ASSERT(cvp, lock, td);
309 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
310 "Waiting on \"%s\"", cvp->cv_description);
311
312 if (SCHEDULER_STOPPED_TD(td))
313 return (0);
314
315 #ifdef KTRACE
316 if (KTRPOINT(td, KTR_CSW)) {
317 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
318 ktrcsw(1, 0, wmesg);
319 } else {
320 wmesg[0] = '\0';
321 }
322 #endif
323
324 class = LOCK_CLASS(lock);
325 lock_state = 0;
326 sleepq_lock(cvp);
327
328 CV_WAITERS_INC(cvp);
329 if (lock == &Giant.lock_object)
330 mtx_assert(&Giant, MA_OWNED);
331 DROP_GIANT();
332
333 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
334 sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
335 if (lock != &Giant.lock_object) {
336 if (class->lc_flags & LC_SLEEPABLE)
337 sleepq_release(cvp);
338 WITNESS_SAVE(lock, lock_witness);
339 lock_state = class->lc_unlock(lock);
340 if (class->lc_flags & LC_SLEEPABLE)
341 sleepq_lock(cvp);
342 }
343 rval = sleepq_timedwait(cvp, 0);
344
345 #ifdef KTRACE
346 if (KTRPOINT(td, KTR_CSW))
347 ktrcsw(0, 0, wmesg);
348 #endif
349 PICKUP_GIANT();
350 if (lock != &Giant.lock_object) {
351 class->lc_lock(lock, lock_state);
352 WITNESS_RESTORE(lock, lock_witness);
353 }
354
355 return (rval);
356 }
357
358 /*
359 * Wait on a condition variable for (at most) the value specified in sbt
360 * argument, allowing interruption by signals.
361 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
362 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
363 * was caught.
364 */
365 int
_cv_timedwait_sig_sbt(struct cv * cvp,struct lock_object * lock,sbintime_t sbt,sbintime_t pr,int flags)366 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
367 sbintime_t sbt, sbintime_t pr, int flags)
368 {
369 WITNESS_SAVE_DECL(lock_witness);
370 #ifdef KTRACE
371 char wmesg[WMESGLEN + 1];
372 #endif
373 struct lock_class *class;
374 struct thread *td;
375 int lock_state, rval;
376
377 td = curthread;
378 CV_ASSERT(cvp, lock, td);
379 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
380 "Waiting on \"%s\"", cvp->cv_description);
381
382 if (SCHEDULER_STOPPED_TD(td))
383 return (0);
384
385 #ifdef KTRACE
386 if (KTRPOINT(td, KTR_CSW)) {
387 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg));
388 ktrcsw(1, 0, wmesg);
389 } else {
390 wmesg[0] = '\0';
391 }
392 #endif
393
394 class = LOCK_CLASS(lock);
395 lock_state = 0;
396 sleepq_lock(cvp);
397
398 CV_WAITERS_INC(cvp);
399 if (lock == &Giant.lock_object)
400 mtx_assert(&Giant, MA_OWNED);
401 DROP_GIANT();
402
403 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
404 SLEEPQ_INTERRUPTIBLE, 0);
405 sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
406 if (lock != &Giant.lock_object) {
407 if (class->lc_flags & LC_SLEEPABLE)
408 sleepq_release(cvp);
409 WITNESS_SAVE(lock, lock_witness);
410 lock_state = class->lc_unlock(lock);
411 if (class->lc_flags & LC_SLEEPABLE)
412 sleepq_lock(cvp);
413 }
414 rval = sleepq_timedwait_sig(cvp, 0);
415
416 #ifdef KTRACE
417 if (KTRPOINT(td, KTR_CSW))
418 ktrcsw(0, 0, wmesg);
419 #endif
420 PICKUP_GIANT();
421 if (lock != &Giant.lock_object) {
422 class->lc_lock(lock, lock_state);
423 WITNESS_RESTORE(lock, lock_witness);
424 }
425
426 return (rval);
427 }
428
429 /*
430 * Signal a condition variable, wakes up one waiting thread. Will also wakeup
431 * the swapper if the process is not in memory, so that it can bring the
432 * sleeping process in. Note that this may also result in additional threads
433 * being made runnable. Should be called with the same mutex as was passed to
434 * cv_wait held.
435 */
436 void
cv_signal(struct cv * cvp)437 cv_signal(struct cv *cvp)
438 {
439
440 if (cvp->cv_waiters == 0)
441 return;
442 sleepq_lock(cvp);
443 if (cvp->cv_waiters == 0) {
444 sleepq_release(cvp);
445 return;
446 }
447 if (cvp->cv_waiters == CV_WAITERS_BOUND && sleepq_lookup(cvp) == NULL) {
448 cvp->cv_waiters = 0;
449 sleepq_release(cvp);
450 } else {
451 if (cvp->cv_waiters < CV_WAITERS_BOUND)
452 cvp->cv_waiters--;
453 if (sleepq_signal(cvp, SLEEPQ_CONDVAR | SLEEPQ_DROP, 0, 0))
454 kick_proc0();
455 }
456 }
457
458 /*
459 * Broadcast a signal to a condition variable. Wakes up all waiting threads.
460 * Should be called with the same mutex as was passed to cv_wait held.
461 */
462 void
cv_broadcastpri(struct cv * cvp,int pri)463 cv_broadcastpri(struct cv *cvp, int pri)
464 {
465 int wakeup_swapper;
466
467 if (cvp->cv_waiters == 0)
468 return;
469 /*
470 * XXX sleepq_broadcast pri argument changed from -1 meaning
471 * no pri to 0 meaning no pri.
472 */
473 wakeup_swapper = 0;
474 if (pri == -1)
475 pri = 0;
476 sleepq_lock(cvp);
477 if (cvp->cv_waiters > 0) {
478 cvp->cv_waiters = 0;
479 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
480 }
481 sleepq_release(cvp);
482 if (wakeup_swapper)
483 kick_proc0();
484 }
485