1 /* $NetBSD: kern_lwp.c,v 1.270 2025/04/11 16:04:43 andvar Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020, 2023
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Nathan J. Williams, and Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Overview
35 *
36 * Lightweight processes (LWPs) are the basic unit or thread of
37 * execution within the kernel. The core state of an LWP is described
38 * by "struct lwp", also known as lwp_t.
39 *
40 * Each LWP is contained within a process (described by "struct proc"),
41 * Every process contains at least one LWP, but may contain more. The
42 * process describes attributes shared among all of its LWPs such as a
43 * private address space, global execution state (stopped, active,
44 * zombie, ...), signal disposition and so on. On a multiprocessor
45 * machine, multiple LWPs be executing concurrently in the kernel.
46 *
47 * Execution states
48 *
49 * At any given time, an LWP has overall state that is described by
50 * lwp::l_stat. The states are broken into two sets below. The first
51 * set is guaranteed to represent the absolute, current state of the
52 * LWP:
53 *
54 * LSONPROC
55 *
56 * On processor: the LWP is executing on a CPU, either in the
57 * kernel or in user space.
58 *
59 * LSRUN
60 *
61 * Runnable: the LWP is parked on a run queue, and may soon be
62 * chosen to run by an idle processor, or by a processor that
63 * has been asked to preempt a currently running but lower
64 * priority LWP.
65 *
66 * LSIDL
67 *
68 * Idle: the LWP has been created but has not yet executed, or
69 * it has ceased executing a unit of work and is waiting to be
70 * started again. This state exists so that the LWP can occupy
71 * a slot in the process & PID table, but without having to
72 * worry about being touched; lookups of the LWP by ID will
73 * fail while in this state. The LWP will become visible for
74 * lookup once its state transitions further. Some special
75 * kernel threads also (ab)use this state to indicate that they
76 * are idle (soft interrupts and idle LWPs).
77 *
78 * LSSUSPENDED:
79 *
80 * Suspended: the LWP has had its execution suspended by
81 * another LWP in the same process using the _lwp_suspend()
82 * system call. User-level LWPs also enter the suspended
83 * state when the system is shutting down.
84 *
85 * The second set represent a "statement of intent" on behalf of the
86 * LWP. The LWP may in fact be executing on a processor, may be
87 * sleeping or idle. It is expected to take the necessary action to
88 * stop executing or become "running" again within a short timeframe.
89 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
90 * Importantly, it indicates that its state is tied to a CPU.
91 *
92 * LSZOMB:
93 *
94 * Dead or dying: the LWP has released most of its resources
95 * and is about to switch away into oblivion, or has already
96 * switched away. When it switches away, its few remaining
97 * resources can be collected.
98 *
99 * LSSLEEP:
100 *
101 * Sleeping: the LWP has entered itself onto a sleep queue, and
102 * has switched away or will switch away shortly to allow other
103 * LWPs to run on the CPU.
104 *
105 * LSSTOP:
106 *
107 * Stopped: the LWP has been stopped as a result of a job
108 * control signal, or as a result of the ptrace() interface.
109 *
110 * Stopped LWPs may run briefly within the kernel to handle
111 * signals that they receive, but will not return to user space
112 * until their process' state is changed away from stopped.
113 *
114 * Single LWPs within a process can not be set stopped
115 * selectively: all actions that can stop or continue LWPs
116 * occur at the process level.
117 *
118 * State transitions
119 *
120 * Note that the LSSTOP state may only be set when returning to
121 * user space in userret(), or when sleeping interruptably. The
122 * LSSUSPENDED state may only be set in userret(). Before setting
123 * those states, we try to ensure that the LWPs will release all
124 * locks that they hold, and at a minimum try to ensure that the
125 * LWP can be set runnable again by a signal.
126 *
127 * LWPs may transition states in the following ways:
128 *
129 * RUN -------> ONPROC ONPROC -----> RUN
130 * > SLEEP
131 * > STOPPED
132 * > SUSPENDED
133 * > ZOMB
134 * > IDL (special cases)
135 *
136 * STOPPED ---> RUN SUSPENDED --> RUN
137 * > SLEEP
138 *
139 * SLEEP -----> ONPROC IDL --------> RUN
140 * > RUN > SUSPENDED
141 * > STOPPED > STOPPED
142 * > ONPROC (special cases)
143 *
144 * Some state transitions are only possible with kernel threads (eg
145 * ONPROC -> IDL) and happen under tightly controlled circumstances
146 * free of unwanted side effects.
147 *
148 * Migration
149 *
150 * Migration of threads from one CPU to another could be performed
151 * internally by the scheduler via sched_takecpu() or sched_catchlwp()
152 * functions. The universal lwp_migrate() function should be used for
153 * any other cases. Subsystems in the kernel must be aware that CPU
154 * of LWP may change, while it is not locked.
155 *
156 * Locking
157 *
158 * The majority of fields in 'struct lwp' are covered by a single,
159 * general spin lock pointed to by lwp::l_mutex. The locks covering
160 * each field are documented in sys/lwp.h.
161 *
162 * State transitions must be made with the LWP's general lock held,
163 * and may cause the LWP's lock pointer to change. Manipulation of
164 * the general lock is not performed directly, but through calls to
165 * lwp_lock(), lwp_unlock() and others. It should be noted that the
166 * adaptive locks are not allowed to be released while the LWP's lock
167 * is being held (unlike for other spin-locks).
168 *
169 * States and their associated locks:
170 *
171 * LSIDL, LSONPROC, LSZOMB, LSSUSPENDED:
172 *
173 * Always covered by spc_lwplock, which protects LWPs not
174 * associated with any other sync object. This is a per-CPU
175 * lock and matches lwp::l_cpu.
176 *
177 * LSRUN:
178 *
179 * Always covered by spc_mutex, which protects the run queues.
180 * This is a per-CPU lock and matches lwp::l_cpu.
181 *
182 * LSSLEEP:
183 *
184 * Covered by a lock associated with the sleep queue (sometimes
185 * a turnstile sleep queue) that the LWP resides on. This can
186 * be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
187 *
188 * LSSTOP:
189 *
190 * If the LWP was previously sleeping (l_wchan != NULL), then
191 * l_mutex references the sleep queue lock. If the LWP was
192 * runnable or on the CPU when halted, or has been removed from
193 * the sleep queue since halted, then the lock is spc_lwplock.
194 *
195 * The lock order is as follows:
196 *
197 * sleepq -> turnstile -> spc_lwplock -> spc_mutex
198 *
199 * Each process has a scheduler state lock (proc::p_lock), and a
200 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
201 * so on. When an LWP is to be entered into or removed from one of the
202 * following states, p_lock must be held and the process wide counters
203 * adjusted:
204 *
205 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
206 *
207 * (But not always for kernel threads. There are some special cases
208 * as mentioned above: soft interrupts, and the idle loops.)
209 *
210 * Note that an LWP is considered running or likely to run soon if in
211 * one of the following states. This affects the value of p_nrlwps:
212 *
213 * LSRUN, LSONPROC, LSSLEEP
214 *
215 * p_lock does not need to be held when transitioning among these
216 * three states, hence p_lock is rarely taken for state transitions.
217 */
218
219 #include <sys/cdefs.h>
220 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.270 2025/04/11 16:04:43 andvar Exp $");
221
222 #include "opt_ddb.h"
223 #include "opt_lockdebug.h"
224 #include "opt_dtrace.h"
225
226 #define _LWP_API_PRIVATE
227
228 #include <sys/param.h>
229
230 #include <sys/atomic.h>
231 #include <sys/cprng.h>
232 #include <sys/cpu.h>
233 #include <sys/dtrace_bsd.h>
234 #include <sys/filedesc.h>
235 #include <sys/fstrans.h>
236 #include <sys/futex.h>
237 #include <sys/intr.h>
238 #include <sys/kauth.h>
239 #include <sys/kcov.h>
240 #include <sys/kmem.h>
241 #include <sys/lockdebug.h>
242 #include <sys/lwpctl.h>
243 #include <sys/msan.h>
244 #include <sys/pool.h>
245 #include <sys/proc.h>
246 #include <sys/pset.h>
247 #include <sys/psref.h>
248 #include <sys/ptrace.h>
249 #include <sys/sdt.h>
250 #include <sys/sleepq.h>
251 #include <sys/syncobj.h>
252 #include <sys/syscall_stats.h>
253 #include <sys/syscallargs.h>
254 #include <sys/sysctl.h>
255 #include <sys/systm.h>
256 #include <sys/uidinfo.h>
257 #include <sys/xcall.h>
258
259 #include <uvm/uvm_extern.h>
260 #include <uvm/uvm_object.h>
261
262 static pool_cache_t lwp_cache __read_mostly;
263 struct lwplist alllwp __cacheline_aligned;
264
265 static int lwp_ctor(void *, void *, int);
266 static void lwp_dtor(void *, void *);
267
268 /* DTrace proc provider probes */
269 SDT_PROVIDER_DEFINE(proc);
270
271 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
272 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
273 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
274
275 struct turnstile turnstile0 __cacheline_aligned;
276 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
277 #ifdef LWP0_CPU_INFO
278 .l_cpu = LWP0_CPU_INFO,
279 #endif
280 #ifdef LWP0_MD_INITIALIZER
281 .l_md = LWP0_MD_INITIALIZER,
282 #endif
283 .l_proc = &proc0,
284 .l_lid = 0, /* we own proc0's slot in the pid table */
285 .l_flag = LW_SYSTEM,
286 .l_stat = LSONPROC,
287 .l_ts = &turnstile0,
288 .l_syncobj = &sched_syncobj,
289 .l_refcnt = 0,
290 .l_priority = PRI_USER + NPRI_USER - 1,
291 .l_inheritedprio = -1,
292 .l_class = SCHED_OTHER,
293 .l_psid = PS_NONE,
294 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
295 .l_name = __UNCONST("swapper"),
296 .l_fd = &filedesc0,
297 };
298
299 static int
lwp_maxlwp(void)300 lwp_maxlwp(void)
301 {
302 /* Assume 1 LWP per 1MiB. */
303 uint64_t lwps_per = ctob(physmem) / (1024 * 1024);
304
305 return MAX(MIN(MAXMAXLWP, lwps_per), MAXLWP);
306 }
307
308 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
309
310 /*
311 * sysctl helper routine for kern.maxlwp. Ensures that the new
312 * values are not too low or too high.
313 */
314 static int
sysctl_kern_maxlwp(SYSCTLFN_ARGS)315 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
316 {
317 int error, nmaxlwp;
318 struct sysctlnode node;
319
320 nmaxlwp = maxlwp;
321 node = *rnode;
322 node.sysctl_data = &nmaxlwp;
323 error = sysctl_lookup(SYSCTLFN_CALL(&node));
324 if (error || newp == NULL)
325 return error;
326
327 if (nmaxlwp < 0 || nmaxlwp >= MAXMAXLWP)
328 return EINVAL;
329 if (nmaxlwp > lwp_maxlwp())
330 return EINVAL;
331 maxlwp = nmaxlwp;
332
333 return 0;
334 }
335
336 static void
sysctl_kern_lwp_setup(void)337 sysctl_kern_lwp_setup(void)
338 {
339 sysctl_createv(NULL, 0, NULL, NULL,
340 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
341 CTLTYPE_INT, "maxlwp",
342 SYSCTL_DESCR("Maximum number of simultaneous threads"),
343 sysctl_kern_maxlwp, 0, NULL, 0,
344 CTL_KERN, CTL_CREATE, CTL_EOL);
345 }
346
347 void
lwpinit(void)348 lwpinit(void)
349 {
350
351 LIST_INIT(&alllwp);
352 lwpinit_specificdata();
353 /*
354 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
355 * calls will exit before memory of LWPs is returned to the pool, where
356 * KVA of LWP structure might be freed and re-used for other purposes.
357 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
358 * callers, therefore a regular passive serialization barrier will
359 * do the job.
360 */
361 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0,
362 PR_PSERIALIZE, "lwppl", NULL, IPL_NONE, lwp_ctor, lwp_dtor, NULL);
363
364 maxlwp = lwp_maxlwp();
365 sysctl_kern_lwp_setup();
366 }
367
368 void
lwp0_init(void)369 lwp0_init(void)
370 {
371 struct lwp *l = &lwp0;
372
373 KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
374
375 LIST_INSERT_HEAD(&alllwp, l, l_list);
376
377 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
378 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
379 cv_init(&l->l_sigcv, "sigwait");
380 cv_init(&l->l_waitcv, "vfork");
381
382 l->l_cred = kauth_cred_hold(proc0.p_cred);
383
384 kdtrace_thread_ctor(NULL, l);
385 lwp_initspecific(l);
386
387 SYSCALL_TIME_LWP_INIT(l);
388 }
389
390 /*
391 * Initialize the non-zeroed portion of an lwp_t.
392 */
393 static int
lwp_ctor(void * arg,void * obj,int flags)394 lwp_ctor(void *arg, void *obj, int flags)
395 {
396 lwp_t *l = obj;
397
398 l->l_stat = LSIDL;
399 l->l_cpu = curcpu();
400 l->l_mutex = l->l_cpu->ci_schedstate.spc_lwplock;
401 l->l_ts = kmem_alloc(sizeof(*l->l_ts), flags == PR_WAITOK ?
402 KM_SLEEP : KM_NOSLEEP);
403
404 if (l->l_ts == NULL) {
405 return ENOMEM;
406 } else {
407 turnstile_ctor(l->l_ts);
408 return 0;
409 }
410 }
411
412 static void
lwp_dtor(void * arg,void * obj)413 lwp_dtor(void *arg, void *obj)
414 {
415 lwp_t *l = obj;
416
417 /*
418 * The value of l->l_cpu must still be valid at this point.
419 */
420 KASSERT(l->l_cpu != NULL);
421
422 /*
423 * We can't return turnstile0 to the pool (it didn't come from it),
424 * so if it comes up just drop it quietly and move on.
425 */
426 if (l->l_ts != &turnstile0)
427 kmem_free(l->l_ts, sizeof(*l->l_ts));
428 }
429
430 /*
431 * Set an LWP suspended.
432 *
433 * Must be called with p_lock held, and the LWP locked. Will unlock the
434 * LWP before return.
435 */
436 int
lwp_suspend(struct lwp * curl,struct lwp * t)437 lwp_suspend(struct lwp *curl, struct lwp *t)
438 {
439 int error;
440
441 KASSERT(mutex_owned(t->l_proc->p_lock));
442 KASSERT(lwp_locked(t, NULL));
443
444 KASSERT(curl != t || curl->l_stat == LSONPROC);
445
446 /*
447 * If the current LWP has been told to exit, we must not suspend anyone
448 * else or deadlock could occur. We won't return to userspace.
449 */
450 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
451 lwp_unlock(t);
452 return (EDEADLK);
453 }
454
455 if ((t->l_flag & LW_DBGSUSPEND) != 0) {
456 lwp_unlock(t);
457 return 0;
458 }
459
460 error = 0;
461
462 switch (t->l_stat) {
463 case LSRUN:
464 case LSONPROC:
465 t->l_flag |= LW_WSUSPEND;
466 lwp_need_userret(t);
467 lwp_unlock(t);
468 break;
469
470 case LSSLEEP:
471 t->l_flag |= LW_WSUSPEND;
472 lwp_need_userret(t);
473
474 /*
475 * Kick the LWP and try to get it to the kernel boundary
476 * so that it will release any locks that it holds.
477 * setrunnable() will release the lock.
478 */
479 if ((t->l_flag & LW_SINTR) != 0)
480 setrunnable(t);
481 else
482 lwp_unlock(t);
483 break;
484
485 case LSSUSPENDED:
486 lwp_unlock(t);
487 break;
488
489 case LSSTOP:
490 t->l_flag |= LW_WSUSPEND;
491 lwp_need_userret(t);
492 setrunnable(t);
493 break;
494
495 case LSIDL:
496 case LSZOMB:
497 error = EINTR; /* It's what Solaris does..... */
498 lwp_unlock(t);
499 break;
500 }
501
502 return (error);
503 }
504
505 /*
506 * Restart a suspended LWP.
507 *
508 * Must be called with p_lock held, and the LWP locked. Will unlock the
509 * LWP before return.
510 */
511 void
lwp_continue(struct lwp * l)512 lwp_continue(struct lwp *l)
513 {
514
515 KASSERT(mutex_owned(l->l_proc->p_lock));
516 KASSERT(lwp_locked(l, NULL));
517
518 /* If rebooting or not suspended, then just bail out. */
519 if ((l->l_flag & LW_WREBOOT) != 0) {
520 lwp_unlock(l);
521 return;
522 }
523
524 l->l_flag &= ~LW_WSUSPEND;
525
526 if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
527 lwp_unlock(l);
528 return;
529 }
530
531 /* setrunnable() will release the lock. */
532 setrunnable(l);
533 }
534
535 /*
536 * Restart a stopped LWP.
537 *
538 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
539 * LWP before return.
540 */
541 void
lwp_unstop(struct lwp * l)542 lwp_unstop(struct lwp *l)
543 {
544 struct proc *p = l->l_proc;
545
546 KASSERT(mutex_owned(&proc_lock));
547 KASSERT(mutex_owned(p->p_lock));
548
549 lwp_lock(l);
550
551 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
552
553 /* If not stopped, then just bail out. */
554 if (l->l_stat != LSSTOP) {
555 lwp_unlock(l);
556 return;
557 }
558
559 p->p_stat = SACTIVE;
560 p->p_sflag &= ~PS_STOPPING;
561
562 if (!p->p_waited)
563 p->p_pptr->p_nstopchild--;
564
565 if (l->l_wchan == NULL) {
566 /* setrunnable() will release the lock. */
567 setrunnable(l);
568 } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
569 /* setrunnable() so we can receive the signal */
570 setrunnable(l);
571 } else {
572 l->l_stat = LSSLEEP;
573 p->p_nrlwps++;
574 lwp_unlock(l);
575 }
576 }
577
578 /*
579 * Wait for an LWP within the current process to exit. If 'lid' is
580 * non-zero, we are waiting for a specific LWP.
581 *
582 * Must be called with p->p_lock held.
583 */
584 int
lwp_wait(struct lwp * l,lwpid_t lid,lwpid_t * departed,bool exiting)585 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
586 {
587 const lwpid_t curlid = l->l_lid;
588 proc_t *p = l->l_proc;
589 lwp_t *l2, *next;
590 int error;
591
592 KASSERT(mutex_owned(p->p_lock));
593
594 p->p_nlwpwait++;
595 l->l_waitingfor = lid;
596
597 for (;;) {
598 int nfound;
599
600 /*
601 * Avoid a race between exit1() and sigexit(): if the
602 * process is dumping core, then we need to bail out: call
603 * into lwp_userret() where we will be suspended until the
604 * deed is done.
605 */
606 if ((p->p_sflag & PS_WCORE) != 0) {
607 mutex_exit(p->p_lock);
608 lwp_userret(l);
609 KASSERT(false);
610 }
611
612 /*
613 * First off, drain any detached LWP that is waiting to be
614 * reaped.
615 */
616 if ((l2 = p->p_zomblwp) != NULL) {
617 p->p_zomblwp = NULL;
618 lwp_free(l2, false, false);/* releases proc mutex */
619 mutex_enter(p->p_lock);
620 continue;
621 }
622
623 /*
624 * Now look for an LWP to collect. If the whole process is
625 * exiting, count detached LWPs as eligible to be collected,
626 * but don't drain them here.
627 */
628 nfound = 0;
629 error = 0;
630
631 /*
632 * If given a specific LID, go via pid_table and make sure
633 * it's not detached.
634 */
635 if (lid != 0) {
636 l2 = proc_find_lwp(p, lid);
637 if (l2 == NULL) {
638 error = ESRCH;
639 break;
640 }
641 KASSERT(l2->l_lid == lid);
642 if ((l2->l_prflag & LPR_DETACHED) != 0) {
643 error = EINVAL;
644 break;
645 }
646 } else {
647 l2 = LIST_FIRST(&p->p_lwps);
648 }
649 for (; l2 != NULL; l2 = next) {
650 next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
651
652 /*
653 * If a specific wait and the target is waiting on
654 * us, then avoid deadlock. This also traps LWPs
655 * that try to wait on themselves.
656 *
657 * Note that this does not handle more complicated
658 * cycles, like: t1 -> t2 -> t3 -> t1. The process
659 * can still be killed so it is not a major problem.
660 */
661 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
662 error = EDEADLK;
663 break;
664 }
665 if (l2 == l)
666 continue;
667 if ((l2->l_prflag & LPR_DETACHED) != 0) {
668 nfound += exiting;
669 continue;
670 }
671 if (lid != 0) {
672 /*
673 * Mark this LWP as the first waiter, if there
674 * is no other.
675 */
676 if (l2->l_waiter == 0)
677 l2->l_waiter = curlid;
678 } else if (l2->l_waiter != 0) {
679 /*
680 * It already has a waiter - so don't
681 * collect it. If the waiter doesn't
682 * grab it we'll get another chance
683 * later.
684 */
685 nfound++;
686 continue;
687 }
688 nfound++;
689
690 /* No need to lock the LWP in order to see LSZOMB. */
691 if (l2->l_stat != LSZOMB)
692 continue;
693
694 /*
695 * We're no longer waiting. Reset the "first waiter"
696 * pointer on the target, in case it was us.
697 */
698 l->l_waitingfor = 0;
699 l2->l_waiter = 0;
700 p->p_nlwpwait--;
701 if (departed)
702 *departed = l2->l_lid;
703 sched_lwp_collect(l2);
704
705 /* lwp_free() releases the proc lock. */
706 lwp_free(l2, false, false);
707 mutex_enter(p->p_lock);
708 return 0;
709 }
710
711 if (error != 0)
712 break;
713 if (nfound == 0) {
714 error = ESRCH;
715 break;
716 }
717
718 /*
719 * Note: since the lock will be dropped, need to restart on
720 * wakeup to run all LWPs again, e.g. there may be new LWPs.
721 */
722 if (exiting) {
723 KASSERT(p->p_nlwps > 1);
724 error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
725 break;
726 }
727
728 /*
729 * Break out if all LWPs are in _lwp_wait(). There are
730 * other ways to hang the process with _lwp_wait(), but the
731 * sleep is interruptable so little point checking for them.
732 */
733 if (p->p_nlwpwait == p->p_nlwps) {
734 error = EDEADLK;
735 break;
736 }
737
738 /*
739 * Sit around and wait for something to happen. We'll be
740 * awoken if any of the conditions examined change: if an
741 * LWP exits, is collected, or is detached.
742 */
743 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
744 break;
745 }
746
747 /*
748 * We didn't find any LWPs to collect, we may have received a
749 * signal, or some other condition has caused us to bail out.
750 *
751 * If waiting on a specific LWP, clear the waiters marker: some
752 * other LWP may want it. Then, kick all the remaining waiters
753 * so that they can re-check for zombies and for deadlock.
754 */
755 if (lid != 0) {
756 l2 = proc_find_lwp(p, lid);
757 KASSERT(l2 == NULL || l2->l_lid == lid);
758
759 if (l2 != NULL && l2->l_waiter == curlid)
760 l2->l_waiter = 0;
761 }
762 p->p_nlwpwait--;
763 l->l_waitingfor = 0;
764 cv_broadcast(&p->p_lwpcv);
765
766 return error;
767 }
768
769 /*
770 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
771 * The new LWP is created in state LSIDL and must be set running,
772 * suspended, or stopped by the caller.
773 */
774 int
lwp_create(lwp_t * l1,proc_t * p2,vaddr_t uaddr,int flags,void * stack,size_t stacksize,void (* func)(void *),void * arg,lwp_t ** rnewlwpp,int sclass,const sigset_t * sigmask,const stack_t * sigstk)775 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
776 void *stack, size_t stacksize, void (*func)(void *), void *arg,
777 lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
778 const stack_t *sigstk)
779 {
780 struct lwp *l2;
781
782 KASSERT(l1 == curlwp || l1->l_proc == &proc0);
783
784 /*
785 * Enforce limits, excluding the first lwp and kthreads. We must
786 * use the process credentials here when adjusting the limit, as
787 * they are what's tied to the accounting entity. However for
788 * authorizing the action, we'll use the LWP's credentials.
789 */
790 mutex_enter(p2->p_lock);
791 if (p2->p_nlwps != 0 && p2 != &proc0) {
792 uid_t uid = kauth_cred_getuid(p2->p_cred);
793 int count = chglwpcnt(uid, 1);
794 if (__predict_false(count >
795 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
796 if (kauth_authorize_process(l1->l_cred,
797 KAUTH_PROCESS_RLIMIT, p2,
798 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
799 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
800 != 0) {
801 (void)chglwpcnt(uid, -1);
802 mutex_exit(p2->p_lock);
803 return EAGAIN;
804 }
805 }
806 }
807
808 /*
809 * First off, reap any detached LWP waiting to be collected.
810 * We can re-use its LWP structure and turnstile.
811 */
812 if ((l2 = p2->p_zomblwp) != NULL) {
813 p2->p_zomblwp = NULL;
814 lwp_free(l2, true, false);
815 /* p2 now unlocked by lwp_free() */
816 KASSERT(l2->l_ts != NULL);
817 KASSERT(l2->l_inheritedprio == -1);
818 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
819 memset(&l2->l_startzero, 0, sizeof(*l2) -
820 offsetof(lwp_t, l_startzero));
821 } else {
822 mutex_exit(p2->p_lock);
823 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
824 memset(&l2->l_startzero, 0, sizeof(*l2) -
825 offsetof(lwp_t, l_startzero));
826 SLIST_INIT(&l2->l_pi_lenders);
827 }
828
829 /*
830 * Because of lockless lookup via pid_table, the LWP can be locked
831 * and inspected briefly even after it's freed, so a few fields are
832 * kept stable.
833 */
834 KASSERT(l2->l_stat == LSIDL);
835 KASSERT(l2->l_cpu != NULL);
836 KASSERT(l2->l_ts != NULL);
837 KASSERT(l2->l_mutex == l2->l_cpu->ci_schedstate.spc_lwplock);
838
839 l2->l_proc = p2;
840 l2->l_refcnt = 0;
841 l2->l_class = sclass;
842
843 /*
844 * Allocate a process ID for this LWP. We need to do this now
845 * while we can still unwind if it fails. Because we're marked
846 * as LSIDL, no lookups by the ID will succeed.
847 *
848 * N.B. this will always succeed for the first LWP in a process,
849 * because proc_alloc_lwpid() will usurp the slot. Also note
850 * that l2->l_proc MUST be valid so that lookups of the proc
851 * will succeed, even if the LWP itself is not visible.
852 */
853 if (__predict_false(proc_alloc_lwpid(p2, l2) == -1)) {
854 pool_cache_put(lwp_cache, l2);
855 return EAGAIN;
856 }
857
858 /*
859 * If vfork(), we want the LWP to run fast and on the same CPU
860 * as its parent, so that it can reuse the VM context and cache
861 * footprint on the local CPU.
862 */
863 l2->l_boostpri = ((flags & LWP_VFORK) ? PRI_KERNEL : PRI_USER);
864 l2->l_priority = l1->l_priority;
865 l2->l_inheritedprio = -1;
866 l2->l_protectprio = -1;
867 l2->l_auxprio = -1;
868 l2->l_flag = 0;
869 l2->l_pflag = LP_MPSAFE;
870 TAILQ_INIT(&l2->l_ld_locks);
871 l2->l_psrefs = 0;
872 kmsan_lwp_alloc(l2);
873
874 /*
875 * For vfork, borrow parent's lwpctl context if it exists.
876 * This also causes us to return via lwp_userret.
877 */
878 if (flags & LWP_VFORK && l1->l_lwpctl) {
879 l2->l_lwpctl = l1->l_lwpctl;
880 l2->l_flag |= LW_LWPCTL;
881 }
882
883 /*
884 * If not the first LWP in the process, grab a reference to the
885 * descriptor table.
886 */
887 l2->l_fd = p2->p_fd;
888 if (p2->p_nlwps != 0) {
889 KASSERT(l1->l_proc == p2);
890 fd_hold(l2);
891 } else {
892 KASSERT(l1->l_proc != p2);
893 }
894
895 if (p2->p_flag & PK_SYSTEM) {
896 /* Mark it as a system LWP. */
897 l2->l_flag |= LW_SYSTEM;
898 }
899
900 kdtrace_thread_ctor(NULL, l2);
901 lwp_initspecific(l2);
902 sched_lwp_fork(l1, l2);
903 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
904 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
905 cv_init(&l2->l_sigcv, "sigwait");
906 cv_init(&l2->l_waitcv, "vfork");
907 l2->l_syncobj = &sched_syncobj;
908 PSREF_DEBUG_INIT_LWP(l2);
909
910 if (rnewlwpp != NULL)
911 *rnewlwpp = l2;
912
913 /*
914 * PCU state needs to be saved before calling uvm_lwp_fork() so that
915 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
916 */
917 pcu_save_all(l1);
918 #if PCU_UNIT_COUNT > 0
919 l2->l_pcu_valid = l1->l_pcu_valid;
920 #endif
921
922 uvm_lwp_setuarea(l2, uaddr);
923 uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
924
925 mutex_enter(p2->p_lock);
926 l2->l_cred = kauth_cred_hold(p2->p_cred);
927 if ((flags & LWP_DETACHED) != 0) {
928 l2->l_prflag = LPR_DETACHED;
929 p2->p_ndlwps++;
930 } else
931 l2->l_prflag = 0;
932
933 if (l1->l_proc == p2) {
934 /*
935 * These flags are set while p_lock is held. Copy with
936 * p_lock held too, so the LWP doesn't sneak into the
937 * process without them being set.
938 */
939 l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
940 } else {
941 /* fork(): pending core/exit doesn't apply to child. */
942 l2->l_flag |= (l1->l_flag & LW_WREBOOT);
943 }
944
945 l2->l_sigstk = *sigstk;
946 l2->l_sigmask = *sigmask;
947 TAILQ_INIT(&l2->l_sigpend.sp_info);
948 sigemptyset(&l2->l_sigpend.sp_set);
949 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
950 p2->p_nlwps++;
951 p2->p_nrlwps++;
952
953 KASSERT(l2->l_affinity == NULL);
954
955 /* Inherit the affinity mask. */
956 if (l1->l_affinity) {
957 /*
958 * Note that we hold the state lock while inheriting
959 * the affinity to avoid race with sched_setaffinity().
960 */
961 lwp_lock(l1);
962 if (l1->l_affinity) {
963 kcpuset_use(l1->l_affinity);
964 l2->l_affinity = l1->l_affinity;
965 }
966 lwp_unlock(l1);
967 }
968
969 /* Ensure a trip through lwp_userret() if needed. */
970 if ((l2->l_flag & LW_USERRET) != 0) {
971 lwp_need_userret(l2);
972 }
973
974 /* This marks the end of the "must be atomic" section. */
975 mutex_exit(p2->p_lock);
976
977 SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
978
979 mutex_enter(&proc_lock);
980 LIST_INSERT_HEAD(&alllwp, l2, l_list);
981 /* Inherit a processor-set */
982 l2->l_psid = l1->l_psid;
983 mutex_exit(&proc_lock);
984
985 SYSCALL_TIME_LWP_INIT(l2);
986
987 if (p2->p_emul->e_lwp_fork)
988 (*p2->p_emul->e_lwp_fork)(l1, l2);
989
990 return (0);
991 }
992
993 /*
994 * Set a new LWP running. If the process is stopping, then the LWP is
995 * created stopped.
996 */
997 void
lwp_start(lwp_t * l,int flags)998 lwp_start(lwp_t *l, int flags)
999 {
1000 proc_t *p = l->l_proc;
1001
1002 mutex_enter(p->p_lock);
1003 lwp_lock(l);
1004 KASSERT(l->l_stat == LSIDL);
1005 if ((flags & LWP_SUSPENDED) != 0) {
1006 /* It'll suspend itself in lwp_userret(). */
1007 l->l_flag |= LW_WSUSPEND;
1008 lwp_need_userret(l);
1009 }
1010 if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
1011 KASSERT(l->l_wchan == NULL);
1012 l->l_stat = LSSTOP;
1013 p->p_nrlwps--;
1014 lwp_unlock(l);
1015 } else {
1016 setrunnable(l);
1017 /* LWP now unlocked */
1018 }
1019 mutex_exit(p->p_lock);
1020 }
1021
1022 /*
1023 * Called by MD code when a new LWP begins execution. Must be called
1024 * with the previous LWP locked (so at splsched), or if there is no
1025 * previous LWP, at splsched.
1026 */
1027 void
lwp_startup(struct lwp * prev,struct lwp * new_lwp)1028 lwp_startup(struct lwp *prev, struct lwp *new_lwp)
1029 {
1030 kmutex_t *lock;
1031
1032 KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
1033 KASSERT(kpreempt_disabled());
1034 KASSERT(prev != NULL);
1035 KASSERT((prev->l_pflag & LP_RUNNING) != 0);
1036 KASSERT(curcpu()->ci_mtx_count == -2);
1037
1038 /*
1039 * Immediately mark the previous LWP as no longer running and
1040 * unlock (to keep lock wait times short as possible). If a
1041 * zombie, don't touch after clearing LP_RUNNING as it could be
1042 * reaped by another CPU. Use atomic_store_release to ensure
1043 * this -- matches atomic_load_acquire in lwp_free.
1044 */
1045 lock = prev->l_mutex;
1046 if (__predict_false(prev->l_stat == LSZOMB)) {
1047 atomic_store_release(&prev->l_pflag,
1048 prev->l_pflag & ~LP_RUNNING);
1049 } else {
1050 prev->l_pflag &= ~LP_RUNNING;
1051 }
1052 mutex_spin_exit(lock);
1053
1054 /* Correct spin mutex count after mi_switch(). */
1055 curcpu()->ci_mtx_count = 0;
1056
1057 /* Install new VM context. */
1058 if (__predict_true(new_lwp->l_proc->p_vmspace)) {
1059 pmap_activate(new_lwp);
1060 }
1061
1062 /* We remain at IPL_SCHED from mi_switch() - reset it. */
1063 spl0();
1064
1065 LOCKDEBUG_BARRIER(NULL, 0);
1066 SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
1067
1068 /* For kthreads, acquire kernel lock if not MPSAFE. */
1069 if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
1070 KERNEL_LOCK(1, new_lwp);
1071 }
1072 }
1073
1074 /*
1075 * Exit an LWP.
1076 *
1077 * *** WARNING *** This can be called with (l != curlwp) in error paths.
1078 */
1079 void
lwp_exit(struct lwp * l)1080 lwp_exit(struct lwp *l)
1081 {
1082 struct proc *p = l->l_proc;
1083 struct lwp *l2;
1084 bool current;
1085
1086 current = (l == curlwp);
1087
1088 KASSERT(current || l->l_stat == LSIDL);
1089 KASSERT(current || l->l_target_cpu == NULL);
1090 KASSERT(p == curproc);
1091
1092 SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
1093
1094 /* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
1095 LOCKDEBUG_BARRIER(NULL, 0);
1096 KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
1097
1098 /*
1099 * If we are the last live LWP in a process, we need to exit the
1100 * entire process. We do so with an exit status of zero, because
1101 * it's a "controlled" exit, and because that's what Solaris does.
1102 *
1103 * We are not quite a zombie yet, but for accounting purposes we
1104 * must increment the count of zombies here.
1105 *
1106 * Note: the last LWP's specificdata will be deleted here.
1107 */
1108 mutex_enter(p->p_lock);
1109 if (p->p_nlwps - p->p_nzlwps == 1) {
1110 KASSERT(current == true);
1111 KASSERT(p != &proc0);
1112 exit1(l, 0, 0);
1113 /* NOTREACHED */
1114 }
1115 p->p_nzlwps++;
1116
1117 /*
1118 * Perform any required thread cleanup. Do this early so
1119 * anyone wanting to look us up with lwp_getref_lwpid() will
1120 * fail to find us before we become a zombie.
1121 *
1122 * N.B. this will unlock p->p_lock on our behalf.
1123 */
1124 lwp_thread_cleanup(l);
1125
1126 if (p->p_emul->e_lwp_exit)
1127 (*p->p_emul->e_lwp_exit)(l);
1128
1129 /* Drop filedesc reference. */
1130 fd_free();
1131
1132 /* Release fstrans private data. */
1133 fstrans_lwp_dtor(l);
1134
1135 /* Delete the specificdata while it's still safe to sleep. */
1136 lwp_finispecific(l);
1137
1138 /*
1139 * Release our cached credentials.
1140 */
1141 kauth_cred_free(l->l_cred);
1142 callout_destroy(&l->l_timeout_ch);
1143
1144 /*
1145 * If traced, report LWP exit event to the debugger.
1146 *
1147 * Remove the LWP from the global list.
1148 * Free its LID from the PID namespace if needed.
1149 */
1150 mutex_enter(&proc_lock);
1151
1152 if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
1153 (PSL_TRACED|PSL_TRACELWP_EXIT)) {
1154 mutex_enter(p->p_lock);
1155 if (ISSET(p->p_sflag, PS_WEXIT)) {
1156 mutex_exit(p->p_lock);
1157 /*
1158 * We are exiting, bail out without informing parent
1159 * about a terminating LWP as it would deadlock.
1160 */
1161 } else {
1162 eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
1163 mutex_enter(&proc_lock);
1164 }
1165 }
1166
1167 LIST_REMOVE(l, l_list);
1168 mutex_exit(&proc_lock);
1169
1170 /*
1171 * Get rid of all references to the LWP that others (e.g. procfs)
1172 * may have, and mark the LWP as a zombie. If the LWP is detached,
1173 * mark it waiting for collection in the proc structure. Note that
1174 * before we can do that, we need to free any other dead, detached
1175 * LWP waiting to meet its maker.
1176 *
1177 * All conditions need to be observed upon under the same hold of
1178 * p_lock, because if the lock is dropped any of them can change.
1179 */
1180 mutex_enter(p->p_lock);
1181 for (;;) {
1182 if (lwp_drainrefs(l))
1183 continue;
1184 if ((l->l_prflag & LPR_DETACHED) != 0) {
1185 if ((l2 = p->p_zomblwp) != NULL) {
1186 p->p_zomblwp = NULL;
1187 lwp_free(l2, false, false);
1188 /* proc now unlocked */
1189 mutex_enter(p->p_lock);
1190 continue;
1191 }
1192 p->p_zomblwp = l;
1193 }
1194 break;
1195 }
1196
1197 /*
1198 * If we find a pending signal for the process and we have been
1199 * asked to check for signals, then we lose: arrange to have
1200 * all other LWPs in the process check for signals.
1201 */
1202 if ((l->l_flag & LW_PENDSIG) != 0 &&
1203 firstsig(&p->p_sigpend.sp_set) != 0) {
1204 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1205 lwp_lock(l2);
1206 signotify(l2);
1207 lwp_unlock(l2);
1208 }
1209 }
1210
1211 /*
1212 * Release any PCU resources before becoming a zombie.
1213 */
1214 pcu_discard_all(l);
1215
1216 lwp_lock(l);
1217 l->l_stat = LSZOMB;
1218 if (l->l_name != NULL) {
1219 strcpy(l->l_name, "(zombie)");
1220 }
1221 lwp_unlock(l);
1222 p->p_nrlwps--;
1223 if (l->l_lwpctl != NULL)
1224 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1225 mutex_exit(p->p_lock);
1226 cv_broadcast(&p->p_lwpcv);
1227
1228 /*
1229 * We can no longer block. At this point, lwp_free() may already
1230 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1231 *
1232 * Free MD LWP resources.
1233 */
1234 cpu_lwp_free(l, 0);
1235
1236 if (current) {
1237 /* Switch away into oblivion. */
1238 lwp_lock(l);
1239 spc_lock(l->l_cpu);
1240 mi_switch(l);
1241 panic("lwp_exit");
1242 }
1243 }
1244
1245 /*
1246 * Free a dead LWP's remaining resources.
1247 *
1248 * XXXLWP limits.
1249 */
1250 void
lwp_free(struct lwp * l,bool recycle,bool last)1251 lwp_free(struct lwp *l, bool recycle, bool last)
1252 {
1253 struct proc *p = l->l_proc;
1254 struct rusage *ru;
1255 ksiginfoq_t kq;
1256
1257 KASSERT(l != curlwp);
1258 KASSERT(last || mutex_owned(p->p_lock));
1259
1260 /*
1261 * We use the process credentials instead of the lwp credentials here
1262 * because the lwp credentials maybe cached (just after a setuid call)
1263 * and we don't want pay for syncing, since the lwp is going away
1264 * anyway
1265 */
1266 if (p != &proc0 && p->p_nlwps != 1)
1267 (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
1268
1269 /*
1270 * In the unlikely event that the LWP is still on the CPU,
1271 * then spin until it has switched away.
1272 *
1273 * atomic_load_acquire matches atomic_store_release in
1274 * lwp_startup and mi_switch.
1275 */
1276 while (__predict_false((atomic_load_acquire(&l->l_pflag) & LP_RUNNING)
1277 != 0)) {
1278 SPINLOCK_BACKOFF_HOOK;
1279 }
1280
1281 /*
1282 * Now that the LWP's known off the CPU, reset its state back to
1283 * LSIDL, which defeats anything that might have gotten a hold on
1284 * the LWP via pid_table before the ID was freed. It's important
1285 * to do this with both the LWP locked and p_lock held.
1286 *
1287 * Also reset the CPU and lock pointer back to curcpu(), since the
1288 * LWP will in all likelyhood be cached with the current CPU in
1289 * lwp_cache when we free it and later allocated from there again
1290 * (avoid incidental lock contention).
1291 */
1292 lwp_lock(l);
1293 l->l_stat = LSIDL;
1294 l->l_cpu = curcpu();
1295 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_lwplock);
1296
1297 /*
1298 * If this was not the last LWP in the process, then adjust counters
1299 * and unlock. This is done differently for the last LWP in exit1().
1300 */
1301 if (!last) {
1302 /*
1303 * Add the LWP's run time to the process' base value.
1304 * This needs to co-incide with coming off p_lwps.
1305 */
1306 bintime_add(&p->p_rtime, &l->l_rtime);
1307 p->p_pctcpu += l->l_pctcpu;
1308 ru = &p->p_stats->p_ru;
1309 ruadd(ru, &l->l_ru);
1310 LIST_REMOVE(l, l_sibling);
1311 p->p_nlwps--;
1312 p->p_nzlwps--;
1313 if ((l->l_prflag & LPR_DETACHED) != 0)
1314 p->p_ndlwps--;
1315 mutex_exit(p->p_lock);
1316
1317 /*
1318 * Have any LWPs sleeping in lwp_wait() recheck for
1319 * deadlock.
1320 */
1321 cv_broadcast(&p->p_lwpcv);
1322
1323 /* Free the LWP ID. */
1324 mutex_enter(&proc_lock);
1325 proc_free_lwpid(p, l->l_lid);
1326 mutex_exit(&proc_lock);
1327 }
1328
1329 /*
1330 * Destroy the LWP's remaining signal information.
1331 */
1332 ksiginfo_queue_init(&kq);
1333 sigclear(&l->l_sigpend, NULL, &kq);
1334 ksiginfo_queue_drain(&kq);
1335 cv_destroy(&l->l_sigcv);
1336 cv_destroy(&l->l_waitcv);
1337
1338 /*
1339 * Free lwpctl structure and affinity.
1340 */
1341 if (l->l_lwpctl) {
1342 lwp_ctl_free(l);
1343 }
1344 if (l->l_affinity) {
1345 kcpuset_unuse(l->l_affinity, NULL);
1346 l->l_affinity = NULL;
1347 }
1348
1349 /*
1350 * Free remaining data structures and the LWP itself unless the
1351 * caller wants to recycle.
1352 */
1353 if (l->l_name != NULL)
1354 kmem_free(l->l_name, MAXCOMLEN);
1355
1356 kmsan_lwp_free(l);
1357 kcov_lwp_free(l);
1358 cpu_lwp_free2(l);
1359 uvm_lwp_exit(l);
1360
1361 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1362 KASSERT(l->l_inheritedprio == -1);
1363 KASSERT(l->l_blcnt == 0);
1364 kdtrace_thread_dtor(NULL, l);
1365 if (!recycle)
1366 pool_cache_put(lwp_cache, l);
1367 }
1368
1369 /*
1370 * Migrate the LWP to the another CPU. Unlocks the LWP.
1371 */
1372 void
lwp_migrate(lwp_t * l,struct cpu_info * tci)1373 lwp_migrate(lwp_t *l, struct cpu_info *tci)
1374 {
1375 struct schedstate_percpu *tspc;
1376 int lstat = l->l_stat;
1377
1378 KASSERT(lwp_locked(l, NULL));
1379 KASSERT(tci != NULL);
1380
1381 /* If LWP is still on the CPU, it must be handled like LSONPROC */
1382 if ((l->l_pflag & LP_RUNNING) != 0) {
1383 lstat = LSONPROC;
1384 }
1385
1386 /*
1387 * The destination CPU could be changed while previous migration
1388 * was not finished.
1389 */
1390 if (l->l_target_cpu != NULL) {
1391 l->l_target_cpu = tci;
1392 lwp_unlock(l);
1393 return;
1394 }
1395
1396 /* Nothing to do if trying to migrate to the same CPU */
1397 if (l->l_cpu == tci) {
1398 lwp_unlock(l);
1399 return;
1400 }
1401
1402 KASSERT(l->l_target_cpu == NULL);
1403 tspc = &tci->ci_schedstate;
1404 switch (lstat) {
1405 case LSRUN:
1406 l->l_target_cpu = tci;
1407 break;
1408 case LSSLEEP:
1409 l->l_cpu = tci;
1410 break;
1411 case LSIDL:
1412 case LSSTOP:
1413 case LSSUSPENDED:
1414 l->l_cpu = tci;
1415 if (l->l_wchan == NULL) {
1416 lwp_unlock_to(l, tspc->spc_lwplock);
1417 return;
1418 }
1419 break;
1420 case LSONPROC:
1421 l->l_target_cpu = tci;
1422 spc_lock(l->l_cpu);
1423 sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
1424 /* spc now unlocked */
1425 break;
1426 }
1427 lwp_unlock(l);
1428 }
1429
1430 #define lwp_find_exclude(l) \
1431 ((l)->l_stat == LSIDL || (l)->l_stat == LSZOMB)
1432
1433 /*
1434 * Find the LWP in the process. Arguments may be zero, in such case,
1435 * the calling process and first LWP in the list will be used.
1436 * On success - returns proc locked.
1437 *
1438 * => pid == 0 -> look in curproc.
1439 * => pid == -1 -> match any proc.
1440 * => otherwise look up the proc.
1441 *
1442 * => lid == 0 -> first LWP in the proc
1443 * => otherwise specific LWP
1444 */
1445 struct lwp *
lwp_find2(pid_t pid,lwpid_t lid)1446 lwp_find2(pid_t pid, lwpid_t lid)
1447 {
1448 proc_t *p;
1449 lwp_t *l;
1450
1451 /* First LWP of specified proc. */
1452 if (lid == 0) {
1453 switch (pid) {
1454 case -1:
1455 /* No lookup keys. */
1456 return NULL;
1457 case 0:
1458 p = curproc;
1459 mutex_enter(p->p_lock);
1460 break;
1461 default:
1462 mutex_enter(&proc_lock);
1463 p = proc_find(pid);
1464 if (__predict_false(p == NULL)) {
1465 mutex_exit(&proc_lock);
1466 return NULL;
1467 }
1468 mutex_enter(p->p_lock);
1469 mutex_exit(&proc_lock);
1470 break;
1471 }
1472 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1473 if (__predict_true(!lwp_find_exclude(l)))
1474 break;
1475 }
1476 goto out;
1477 }
1478
1479 l = proc_find_lwp_acquire_proc(lid, &p);
1480 if (l == NULL)
1481 return NULL;
1482 KASSERT(p != NULL);
1483 KASSERT(mutex_owned(p->p_lock));
1484
1485 if (__predict_false(lwp_find_exclude(l))) {
1486 l = NULL;
1487 goto out;
1488 }
1489
1490 /* Apply proc filter, if applicable. */
1491 switch (pid) {
1492 case -1:
1493 /* Match anything. */
1494 break;
1495 case 0:
1496 if (p != curproc)
1497 l = NULL;
1498 break;
1499 default:
1500 if (p->p_pid != pid)
1501 l = NULL;
1502 break;
1503 }
1504
1505 out:
1506 if (__predict_false(l == NULL)) {
1507 mutex_exit(p->p_lock);
1508 }
1509 return l;
1510 }
1511
1512 /*
1513 * Look up a live LWP within the specified process.
1514 *
1515 * Must be called with p->p_lock held (as it looks at the radix tree,
1516 * and also wants to exclude idle and zombie LWPs).
1517 */
1518 struct lwp *
lwp_find(struct proc * p,lwpid_t id)1519 lwp_find(struct proc *p, lwpid_t id)
1520 {
1521 struct lwp *l;
1522
1523 KASSERT(mutex_owned(p->p_lock));
1524
1525 l = proc_find_lwp(p, id);
1526 KASSERT(l == NULL || l->l_lid == id);
1527
1528 /*
1529 * No need to lock - all of these conditions will
1530 * be visible with the process level mutex held.
1531 */
1532 if (__predict_false(l != NULL && lwp_find_exclude(l)))
1533 l = NULL;
1534
1535 return l;
1536 }
1537
1538 /*
1539 * Verify that an LWP is locked, and optionally verify that the lock matches
1540 * one we specify.
1541 */
1542 int
lwp_locked(struct lwp * l,kmutex_t * mtx)1543 lwp_locked(struct lwp *l, kmutex_t *mtx)
1544 {
1545 kmutex_t *cur = l->l_mutex;
1546
1547 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1548 }
1549
1550 /*
1551 * Lend a new mutex to an LWP. The old mutex must be held.
1552 */
1553 kmutex_t *
lwp_setlock(struct lwp * l,kmutex_t * mtx)1554 lwp_setlock(struct lwp *l, kmutex_t *mtx)
1555 {
1556 kmutex_t *oldmtx = l->l_mutex;
1557
1558 KASSERT(mutex_owned(oldmtx));
1559
1560 atomic_store_release(&l->l_mutex, mtx);
1561 return oldmtx;
1562 }
1563
1564 /*
1565 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1566 * must be held.
1567 */
1568 void
lwp_unlock_to(struct lwp * l,kmutex_t * mtx)1569 lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
1570 {
1571 kmutex_t *old;
1572
1573 KASSERT(lwp_locked(l, NULL));
1574
1575 old = l->l_mutex;
1576 atomic_store_release(&l->l_mutex, mtx);
1577 mutex_spin_exit(old);
1578 }
1579
1580 int
lwp_trylock(struct lwp * l)1581 lwp_trylock(struct lwp *l)
1582 {
1583 kmutex_t *old;
1584
1585 for (;;) {
1586 if (!mutex_tryenter(old = atomic_load_consume(&l->l_mutex)))
1587 return 0;
1588 if (__predict_true(atomic_load_relaxed(&l->l_mutex) == old))
1589 return 1;
1590 mutex_spin_exit(old);
1591 }
1592 }
1593
1594 void
lwp_unsleep(lwp_t * l,bool unlock)1595 lwp_unsleep(lwp_t *l, bool unlock)
1596 {
1597
1598 KASSERT(mutex_owned(l->l_mutex));
1599 (*l->l_syncobj->sobj_unsleep)(l, unlock);
1600 }
1601
1602 /*
1603 * Lock an LWP.
1604 */
1605 void
lwp_lock(lwp_t * l)1606 lwp_lock(lwp_t *l)
1607 {
1608 kmutex_t *old = atomic_load_consume(&l->l_mutex);
1609
1610 /*
1611 * Note: mutex_spin_enter() will have posted a read barrier.
1612 * Re-test l->l_mutex. If it has changed, we need to try again.
1613 */
1614 mutex_spin_enter(old);
1615 while (__predict_false(atomic_load_relaxed(&l->l_mutex) != old)) {
1616 mutex_spin_exit(old);
1617 old = atomic_load_consume(&l->l_mutex);
1618 mutex_spin_enter(old);
1619 }
1620 }
1621
1622 /*
1623 * Unlock an LWP.
1624 */
1625 void
lwp_unlock(lwp_t * l)1626 lwp_unlock(lwp_t *l)
1627 {
1628
1629 mutex_spin_exit(l->l_mutex);
1630 }
1631
1632 void
lwp_changepri(lwp_t * l,pri_t pri)1633 lwp_changepri(lwp_t *l, pri_t pri)
1634 {
1635
1636 KASSERT(mutex_owned(l->l_mutex));
1637
1638 if (l->l_priority == pri)
1639 return;
1640
1641 (*l->l_syncobj->sobj_changepri)(l, pri);
1642 KASSERT(l->l_priority == pri);
1643 }
1644
1645 void
lwp_lendpri(lwp_t * l,pri_t pri)1646 lwp_lendpri(lwp_t *l, pri_t pri)
1647 {
1648 KASSERT(mutex_owned(l->l_mutex));
1649
1650 (*l->l_syncobj->sobj_lendpri)(l, pri);
1651 KASSERT(l->l_inheritedprio == pri);
1652 }
1653
1654 pri_t
lwp_eprio(lwp_t * l)1655 lwp_eprio(lwp_t *l)
1656 {
1657 pri_t pri = l->l_priority;
1658
1659 KASSERT(mutex_owned(l->l_mutex));
1660
1661 /*
1662 * Timeshared/user LWPs get a temporary priority boost for blocking
1663 * in kernel. This is key to good interactive response on a loaded
1664 * system: without it, things will seem very sluggish to the user.
1665 *
1666 * The function of the boost is to get the LWP onto a CPU and
1667 * running quickly. Once that happens the LWP loses the priority
1668 * boost and could be preempted very quickly by another LWP but that
1669 * won't happen often enough to be an annoyance.
1670 */
1671 if (pri <= MAXPRI_USER && l->l_boostpri > MAXPRI_USER)
1672 pri = (pri >> 1) + l->l_boostpri;
1673
1674 return MAX(l->l_auxprio, pri);
1675 }
1676
1677 /*
1678 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1679 * set or a preemption is required.
1680 */
1681 void
lwp_userret(struct lwp * l)1682 lwp_userret(struct lwp *l)
1683 {
1684 struct proc *p;
1685 int sig, f;
1686
1687 KASSERT(l == curlwp);
1688 KASSERT(l->l_stat == LSONPROC);
1689 p = l->l_proc;
1690
1691 for (;;) {
1692 /*
1693 * This is the main location that user preemptions are
1694 * processed.
1695 */
1696 preempt_point();
1697
1698 /*
1699 * It is safe to do this unlocked and without raised SPL,
1700 * since whenever a flag of interest is added to l_flag the
1701 * LWP will take an AST and come down this path again. If a
1702 * remote CPU posts the AST, it will be done with an IPI
1703 * (strongly synchronising).
1704 */
1705 if ((f = atomic_load_relaxed(&l->l_flag) & LW_USERRET) == 0) {
1706 return;
1707 }
1708
1709 /*
1710 * Start out with the correct credentials.
1711 */
1712 if ((f & LW_CACHECRED) != 0) {
1713 kauth_cred_t oc = l->l_cred;
1714 mutex_enter(p->p_lock);
1715 l->l_cred = kauth_cred_hold(p->p_cred);
1716 lwp_lock(l);
1717 l->l_flag &= ~LW_CACHECRED;
1718 lwp_unlock(l);
1719 mutex_exit(p->p_lock);
1720 kauth_cred_free(oc);
1721 }
1722
1723 /*
1724 * Process pending signals first, unless the process
1725 * is dumping core or exiting, where we will instead
1726 * enter the LW_WSUSPEND case below.
1727 */
1728 if ((f & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == LW_PENDSIG) {
1729 mutex_enter(p->p_lock);
1730 while ((sig = issignal(l)) != 0)
1731 postsig(sig);
1732 mutex_exit(p->p_lock);
1733 continue;
1734 }
1735
1736 /*
1737 * Core-dump or suspend pending.
1738 *
1739 * In case of core dump, suspend ourselves, so that the kernel
1740 * stack and therefore the userland registers saved in the
1741 * trapframe are around for coredump() to write them out.
1742 * We also need to save any PCU resources that we have so that
1743 * they accessible for coredump(). We issue a wakeup on
1744 * p->p_lwpcv so that sigexit() will write the core file out
1745 * once all other LWPs are suspended.
1746 */
1747 if ((f & LW_WSUSPEND) != 0) {
1748 pcu_save_all(l);
1749 mutex_enter(p->p_lock);
1750 p->p_nrlwps--;
1751 lwp_lock(l);
1752 l->l_stat = LSSUSPENDED;
1753 lwp_unlock(l);
1754 mutex_exit(p->p_lock);
1755 cv_broadcast(&p->p_lwpcv);
1756 lwp_lock(l);
1757 spc_lock(l->l_cpu);
1758 mi_switch(l);
1759 continue;
1760 }
1761
1762 /*
1763 * Process is exiting. The core dump and signal cases must
1764 * be handled first.
1765 */
1766 if ((f & LW_WEXIT) != 0) {
1767 lwp_exit(l);
1768 KASSERT(0);
1769 /* NOTREACHED */
1770 }
1771
1772 /*
1773 * Update lwpctl processor (for vfork child_return).
1774 */
1775 if ((f & LW_LWPCTL) != 0) {
1776 lwp_lock(l);
1777 KASSERT(kpreempt_disabled());
1778 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1779 l->l_lwpctl->lc_pctr++;
1780 l->l_flag &= ~LW_LWPCTL;
1781 lwp_unlock(l);
1782 continue;
1783 }
1784 }
1785 }
1786
1787 /*
1788 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1789 */
1790 void
lwp_need_userret(struct lwp * l)1791 lwp_need_userret(struct lwp *l)
1792 {
1793
1794 KASSERT(!cpu_intr_p());
1795 KASSERT(lwp_locked(l, NULL) || l->l_stat == LSIDL);
1796
1797 /*
1798 * If the LWP is in any state other than LSONPROC, we know that it
1799 * is executing in-kernel and will hit userret() on the way out.
1800 *
1801 * If the LWP is curlwp, then we know we'll be back out to userspace
1802 * soon (can't be called from a hardware interrupt here).
1803 *
1804 * Otherwise, we can't be sure what the LWP is doing, so first make
1805 * sure the update to l_flag will be globally visible, and then
1806 * force the LWP to take a trip through trap() where it will do
1807 * userret().
1808 */
1809 if (l->l_stat == LSONPROC && l != curlwp) {
1810 membar_producer();
1811 cpu_signotify(l);
1812 }
1813 }
1814
1815 /*
1816 * Add one reference to an LWP. This will prevent the LWP from
1817 * exiting, thus keep the lwp structure and PCB around to inspect.
1818 */
1819 void
lwp_addref(struct lwp * l)1820 lwp_addref(struct lwp *l)
1821 {
1822 KASSERT(mutex_owned(l->l_proc->p_lock));
1823 KASSERT(l->l_stat != LSZOMB);
1824 l->l_refcnt++;
1825 }
1826
1827 /*
1828 * Remove one reference to an LWP. If this is the last reference,
1829 * then we must finalize the LWP's death.
1830 */
1831 void
lwp_delref(struct lwp * l)1832 lwp_delref(struct lwp *l)
1833 {
1834 struct proc *p = l->l_proc;
1835
1836 mutex_enter(p->p_lock);
1837 lwp_delref2(l);
1838 mutex_exit(p->p_lock);
1839 }
1840
1841 /*
1842 * Remove one reference to an LWP. If this is the last reference,
1843 * then we must finalize the LWP's death. The proc mutex is held
1844 * on entry.
1845 */
1846 void
lwp_delref2(struct lwp * l)1847 lwp_delref2(struct lwp *l)
1848 {
1849 struct proc *p = l->l_proc;
1850
1851 KASSERT(mutex_owned(p->p_lock));
1852 KASSERT(l->l_stat != LSZOMB);
1853 KASSERT(l->l_refcnt > 0);
1854
1855 if (--l->l_refcnt == 0)
1856 cv_broadcast(&p->p_lwpcv);
1857 }
1858
1859 /*
1860 * Drain all references to the current LWP. Returns true if
1861 * we blocked.
1862 */
1863 bool
lwp_drainrefs(struct lwp * l)1864 lwp_drainrefs(struct lwp *l)
1865 {
1866 struct proc *p = l->l_proc;
1867 bool rv = false;
1868
1869 KASSERT(mutex_owned(p->p_lock));
1870
1871 l->l_prflag |= LPR_DRAINING;
1872
1873 while (l->l_refcnt > 0) {
1874 rv = true;
1875 cv_wait(&p->p_lwpcv, p->p_lock);
1876 }
1877 return rv;
1878 }
1879
1880 /*
1881 * Return true if the specified LWP is 'alive'. Only p->p_lock need
1882 * be held.
1883 */
1884 bool
lwp_alive(lwp_t * l)1885 lwp_alive(lwp_t *l)
1886 {
1887
1888 KASSERT(mutex_owned(l->l_proc->p_lock));
1889
1890 switch (l->l_stat) {
1891 case LSSLEEP:
1892 case LSRUN:
1893 case LSONPROC:
1894 case LSSTOP:
1895 case LSSUSPENDED:
1896 return true;
1897 default:
1898 return false;
1899 }
1900 }
1901
1902 /*
1903 * Return first live LWP in the process.
1904 */
1905 lwp_t *
lwp_find_first(proc_t * p)1906 lwp_find_first(proc_t *p)
1907 {
1908 lwp_t *l;
1909
1910 KASSERT(mutex_owned(p->p_lock));
1911
1912 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1913 if (lwp_alive(l)) {
1914 return l;
1915 }
1916 }
1917
1918 return NULL;
1919 }
1920
1921 /*
1922 * Allocate a new lwpctl structure for a user LWP.
1923 */
1924 int
lwp_ctl_alloc(vaddr_t * uaddr)1925 lwp_ctl_alloc(vaddr_t *uaddr)
1926 {
1927 lcproc_t *lp;
1928 u_int bit, i, offset;
1929 struct uvm_object *uao;
1930 int error;
1931 lcpage_t *lcp;
1932 proc_t *p;
1933 lwp_t *l;
1934
1935 l = curlwp;
1936 p = l->l_proc;
1937
1938 /* don't allow a vforked process to create lwp ctls */
1939 if (p->p_lflag & PL_PPWAIT)
1940 return EBUSY;
1941
1942 if (l->l_lcpage != NULL) {
1943 lcp = l->l_lcpage;
1944 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1945 return 0;
1946 }
1947
1948 /* First time around, allocate header structure for the process. */
1949 if ((lp = p->p_lwpctl) == NULL) {
1950 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1951 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1952 lp->lp_uao = NULL;
1953 TAILQ_INIT(&lp->lp_pages);
1954 mutex_enter(p->p_lock);
1955 if (p->p_lwpctl == NULL) {
1956 p->p_lwpctl = lp;
1957 mutex_exit(p->p_lock);
1958 } else {
1959 mutex_exit(p->p_lock);
1960 mutex_destroy(&lp->lp_lock);
1961 kmem_free(lp, sizeof(*lp));
1962 lp = p->p_lwpctl;
1963 }
1964 }
1965
1966 /*
1967 * Set up an anonymous memory region to hold the shared pages.
1968 * Map them into the process' address space. The user vmspace
1969 * gets the first reference on the UAO.
1970 */
1971 mutex_enter(&lp->lp_lock);
1972 if (lp->lp_uao == NULL) {
1973 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1974 lp->lp_cur = 0;
1975 lp->lp_max = LWPCTL_UAREA_SZ;
1976 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1977 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
1978 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1979 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1980 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1981 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1982 if (error != 0) {
1983 uao_detach(lp->lp_uao);
1984 lp->lp_uao = NULL;
1985 mutex_exit(&lp->lp_lock);
1986 return error;
1987 }
1988 }
1989
1990 /* Get a free block and allocate for this LWP. */
1991 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1992 if (lcp->lcp_nfree != 0)
1993 break;
1994 }
1995 if (lcp == NULL) {
1996 /* Nothing available - try to set up a free page. */
1997 if (lp->lp_cur == lp->lp_max) {
1998 mutex_exit(&lp->lp_lock);
1999 return ENOMEM;
2000 }
2001 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
2002
2003 /*
2004 * Wire the next page down in kernel space. Since this
2005 * is a new mapping, we must add a reference.
2006 */
2007 uao = lp->lp_uao;
2008 (*uao->pgops->pgo_reference)(uao);
2009 lcp->lcp_kaddr = vm_map_min(kernel_map);
2010 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
2011 uao, lp->lp_cur, PAGE_SIZE,
2012 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
2013 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
2014 if (error != 0) {
2015 mutex_exit(&lp->lp_lock);
2016 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2017 (*uao->pgops->pgo_detach)(uao);
2018 return error;
2019 }
2020 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
2021 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
2022 if (error != 0) {
2023 mutex_exit(&lp->lp_lock);
2024 uvm_unmap(kernel_map, lcp->lcp_kaddr,
2025 lcp->lcp_kaddr + PAGE_SIZE);
2026 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2027 return error;
2028 }
2029 /* Prepare the page descriptor and link into the list. */
2030 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
2031 lp->lp_cur += PAGE_SIZE;
2032 lcp->lcp_nfree = LWPCTL_PER_PAGE;
2033 lcp->lcp_rotor = 0;
2034 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
2035 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2036 }
2037 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
2038 if (++i >= LWPCTL_BITMAP_ENTRIES)
2039 i = 0;
2040 }
2041 bit = ffs(lcp->lcp_bitmap[i]) - 1;
2042 lcp->lcp_bitmap[i] ^= (1U << bit);
2043 lcp->lcp_rotor = i;
2044 lcp->lcp_nfree--;
2045 l->l_lcpage = lcp;
2046 offset = (i << 5) + bit;
2047 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
2048 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
2049 mutex_exit(&lp->lp_lock);
2050
2051 KPREEMPT_DISABLE(l);
2052 l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
2053 KPREEMPT_ENABLE(l);
2054
2055 return 0;
2056 }
2057
2058 /*
2059 * Free an lwpctl structure back to the per-process list.
2060 */
2061 void
lwp_ctl_free(lwp_t * l)2062 lwp_ctl_free(lwp_t *l)
2063 {
2064 struct proc *p = l->l_proc;
2065 lcproc_t *lp;
2066 lcpage_t *lcp;
2067 u_int map, offset;
2068
2069 /* don't free a lwp context we borrowed for vfork */
2070 if (p->p_lflag & PL_PPWAIT) {
2071 l->l_lwpctl = NULL;
2072 return;
2073 }
2074
2075 lp = p->p_lwpctl;
2076 KASSERT(lp != NULL);
2077
2078 lcp = l->l_lcpage;
2079 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
2080 KASSERT(offset < LWPCTL_PER_PAGE);
2081
2082 mutex_enter(&lp->lp_lock);
2083 lcp->lcp_nfree++;
2084 map = offset >> 5;
2085 lcp->lcp_bitmap[map] |= (1U << (offset & 31));
2086 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
2087 lcp->lcp_rotor = map;
2088 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
2089 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
2090 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2091 }
2092 mutex_exit(&lp->lp_lock);
2093 }
2094
2095 /*
2096 * Process is exiting; tear down lwpctl state. This can only be safely
2097 * called by the last LWP in the process.
2098 */
2099 void
lwp_ctl_exit(void)2100 lwp_ctl_exit(void)
2101 {
2102 lcpage_t *lcp, *next;
2103 lcproc_t *lp;
2104 proc_t *p;
2105 lwp_t *l;
2106
2107 l = curlwp;
2108 l->l_lwpctl = NULL;
2109 l->l_lcpage = NULL;
2110 p = l->l_proc;
2111 lp = p->p_lwpctl;
2112
2113 KASSERT(lp != NULL);
2114 KASSERT(p->p_nlwps == 1);
2115
2116 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
2117 next = TAILQ_NEXT(lcp, lcp_chain);
2118 uvm_unmap(kernel_map, lcp->lcp_kaddr,
2119 lcp->lcp_kaddr + PAGE_SIZE);
2120 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2121 }
2122
2123 if (lp->lp_uao != NULL) {
2124 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
2125 lp->lp_uva + LWPCTL_UAREA_SZ);
2126 }
2127
2128 mutex_destroy(&lp->lp_lock);
2129 kmem_free(lp, sizeof(*lp));
2130 p->p_lwpctl = NULL;
2131 }
2132
2133 /*
2134 * Return the current LWP's "preemption counter". Used to detect
2135 * preemption across operations that can tolerate preemption without
2136 * crashing, but which may generate incorrect results if preempted.
2137 *
2138 * We do arithmetic in unsigned long to avoid undefined behaviour in
2139 * the event of arithmetic overflow on LP32, and issue __insn_barrier()
2140 * on both sides so this can safely be used to detect changes to the
2141 * preemption counter in loops around other memory accesses even in the
2142 * event of whole-program optimization (e.g., gcc -flto).
2143 */
2144 long
lwp_pctr(void)2145 lwp_pctr(void)
2146 {
2147 unsigned long pctr;
2148
2149 __insn_barrier();
2150 pctr = curlwp->l_ru.ru_nvcsw;
2151 pctr += curlwp->l_ru.ru_nivcsw;
2152 __insn_barrier();
2153 return pctr;
2154 }
2155
2156 /*
2157 * Set an LWP's private data pointer.
2158 */
2159 int
lwp_setprivate(struct lwp * l,void * ptr)2160 lwp_setprivate(struct lwp *l, void *ptr)
2161 {
2162 int error = 0;
2163
2164 l->l_private = ptr;
2165 #ifdef __HAVE_CPU_LWP_SETPRIVATE
2166 error = cpu_lwp_setprivate(l, ptr);
2167 #endif
2168 return error;
2169 }
2170
2171 /*
2172 * Perform any thread-related cleanup on LWP exit.
2173 * N.B. l->l_proc->p_lock must be HELD on entry but will
2174 * be released before returning!
2175 */
2176 void
lwp_thread_cleanup(struct lwp * l)2177 lwp_thread_cleanup(struct lwp *l)
2178 {
2179
2180 KASSERT(mutex_owned(l->l_proc->p_lock));
2181 mutex_exit(l->l_proc->p_lock);
2182
2183 /*
2184 * If the LWP has robust futexes, release them all
2185 * now.
2186 */
2187 if (__predict_false(l->l_robust_head != 0)) {
2188 futex_release_all_lwp(l);
2189 }
2190 }
2191
2192 #if defined(DDB)
2193 #include <machine/pcb.h>
2194
2195 void
lwp_whatis(uintptr_t addr,void (* pr)(const char *,...))2196 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
2197 {
2198 lwp_t *l;
2199
2200 LIST_FOREACH(l, &alllwp, l_list) {
2201 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
2202
2203 if (addr < stack || stack + KSTACK_SIZE <= addr) {
2204 continue;
2205 }
2206 (*pr)("%p is %p+%zu, LWP %p's stack\n",
2207 (void *)addr, (void *)stack,
2208 (size_t)(addr - stack), l);
2209 }
2210 }
2211 #endif /* defined(DDB) */
2212