1 /* $NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Generic software interrupt framework.
34 *
35 * Overview
36 *
37 * The soft interrupt framework provides a mechanism to schedule a
38 * low priority callback that runs with thread context. It allows
39 * for dynamic registration of software interrupts, and for fair
40 * queueing and prioritization of those interrupts. The callbacks
41 * can be scheduled to run from nearly any point in the kernel: by
42 * code running with thread context, by code running from a
43 * hardware interrupt handler, and at any interrupt priority
44 * level.
45 *
46 * Priority levels
47 *
48 * Since soft interrupt dispatch can be tied to the underlying
49 * architecture's interrupt dispatch code, it can be limited
50 * both by the capabilities of the hardware and the capabilities
51 * of the interrupt dispatch code itself. The number of priority
52 * levels is restricted to four. In order of priority (lowest to
53 * highest) the levels are: clock, bio, net, serial.
54 *
55 * The names are symbolic and in isolation do not have any direct
56 * connection with a particular kind of device activity: they are
57 * only meant as a guide.
58 *
59 * The four priority levels map directly to scheduler priority
60 * levels, and where the architecture implements 'fast' software
61 * interrupts, they also map onto interrupt priorities. The
62 * interrupt priorities are intended to be hidden from machine
63 * independent code, which should use thread-safe mechanisms to
64 * synchronize with software interrupts (for example: mutexes).
65 *
66 * Capabilities
67 *
68 * Software interrupts run with limited machine context. In
69 * particular, they do not posess any address space context. They
70 * should not try to operate on user space addresses, or to use
71 * virtual memory facilities other than those noted as interrupt
72 * safe.
73 *
74 * Unlike hardware interrupts, software interrupts do have thread
75 * context. They may block on synchronization objects, sleep, and
76 * resume execution at a later time.
77 *
78 * Since software interrupts are a limited resource and run with
79 * higher priority than most other LWPs in the system, all
80 * block-and-resume activity by a software interrupt must be kept
81 * short to allow further processing at that level to continue. By
82 * extension, code running with process context must take care to
83 * ensure that any lock that may be taken from a software interrupt
84 * can not be held for more than a short period of time.
85 *
86 * The kernel does not allow software interrupts to use facilities
87 * or perform actions that may block for a significant amount of
88 * time. This means that it's not valid for a software interrupt
89 * to sleep on condition variables or wait for resources to become
90 * available (for example, memory).
91 *
92 * Per-CPU operation
93 *
94 * If a soft interrupt is triggered on a CPU, it can only be
95 * dispatched on the same CPU. Each LWP dedicated to handling a
96 * soft interrupt is bound to its home CPU, so if the LWP blocks
97 * and needs to run again, it can only run there. Nearly all data
98 * structures used to manage software interrupts are per-CPU.
99 *
100 * The per-CPU requirement is intended to reduce "ping-pong" of
101 * cache lines between CPUs: lines occupied by data structures
102 * used to manage the soft interrupts, and lines occupied by data
103 * items being passed down to the soft interrupt. As a positive
104 * side effect, this also means that the soft interrupt dispatch
105 * code does not need to to use spinlocks to synchronize.
106 *
107 * Generic implementation
108 *
109 * A generic, low performance implementation is provided that
110 * works across all architectures, with no machine-dependent
111 * modifications needed. This implementation uses the scheduler,
112 * and so has a number of restrictions:
113 *
114 * 1) The software interrupts are not currently preemptive, so
115 * must wait for the currently executing LWP to yield the CPU.
116 * This can introduce latency.
117 *
118 * 2) An expensive context switch is required for a software
119 * interrupt to be handled.
120 *
121 * 'Fast' software interrupts
122 *
123 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements
124 * the fast mechanism. Threads running either in the kernel or in
125 * userspace will be interrupted, but will not be preempted. When
126 * the soft interrupt completes execution, the interrupted LWP
127 * is resumed. Interrupt dispatch code must provide the minimum
128 * level of context necessary for the soft interrupt to block and
129 * be resumed at a later time. The machine-dependent dispatch
130 * path looks something like the following:
131 *
132 * softintr()
133 * {
134 * go to IPL_HIGH if necessary for switch;
135 * save any necessary registers in a format that can be
136 * restored by cpu_switchto if the softint blocks;
137 * arrange for cpu_switchto() to restore into the
138 * trampoline function;
139 * identify LWP to handle this interrupt;
140 * switch to the LWP's stack;
141 * switch register stacks, if necessary;
142 * assign new value of curlwp;
143 * call MI softint_dispatch, passing old curlwp and IPL
144 * to execute interrupt at;
145 * switch back to old stack;
146 * switch back to old register stack, if necessary;
147 * restore curlwp;
148 * return to interrupted LWP;
149 * }
150 *
151 * If the soft interrupt blocks, a trampoline function is returned
152 * to in the context of the interrupted LWP, as arranged for by
153 * softint():
154 *
155 * softint_ret()
156 * {
157 * unlock soft interrupt LWP;
158 * resume interrupt processing, likely returning to
159 * interrupted LWP or dispatching another, different
160 * interrupt;
161 * }
162 *
163 * Once the soft interrupt has fired (and even if it has blocked),
164 * no further soft interrupts at that level will be triggered by
165 * MI code until the soft interrupt handler has ceased execution.
166 * If a soft interrupt handler blocks and is resumed, it resumes
167 * execution as a normal LWP (kthread) and gains VM context. Only
168 * when it has completed and is ready to fire again will it
169 * interrupt other threads.
170 */
171
172 #include <sys/cdefs.h>
173 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $");
174
175 #include <sys/param.h>
176 #include <sys/proc.h>
177 #include <sys/intr.h>
178 #include <sys/ipi.h>
179 #include <sys/lock.h>
180 #include <sys/mutex.h>
181 #include <sys/kernel.h>
182 #include <sys/kthread.h>
183 #include <sys/evcnt.h>
184 #include <sys/cpu.h>
185 #include <sys/xcall.h>
186 #include <sys/psref.h>
187 #include <sys/sdt.h>
188
189 #include <uvm/uvm_extern.h>
190
191 /* This could overlap with signal info in struct lwp. */
192 typedef struct softint {
193 SIMPLEQ_HEAD(, softhand) si_q;
194 struct lwp *si_lwp;
195 struct cpu_info *si_cpu;
196 uintptr_t si_machdep;
197 struct evcnt si_evcnt;
198 struct evcnt si_evcnt_block;
199 volatile int si_active;
200 int si_ipl;
201 char si_name[8];
202 char si_name_block[8+6];
203 } softint_t;
204
205 typedef struct softhand {
206 SIMPLEQ_ENTRY(softhand) sh_q;
207 void (*sh_func)(void *);
208 void *sh_arg;
209 softint_t *sh_isr;
210 u_int sh_flags;
211 u_int sh_ipi_id;
212 } softhand_t;
213
214 typedef struct softcpu {
215 struct cpu_info *sc_cpu;
216 softint_t sc_int[SOFTINT_COUNT];
217 softhand_t sc_hand[1];
218 } softcpu_t;
219
220 static void softint_thread(void *);
221
222 u_int softint_bytes = 32768;
223 u_int softint_timing;
224 static u_int softint_max;
225 static kmutex_t softint_lock;
226
227 SDT_PROBE_DEFINE4(sdt, kernel, softint, establish,
228 "void *"/*sih*/,
229 "void (*)(void *)"/*func*/,
230 "void *"/*arg*/,
231 "unsigned"/*flags*/);
232
233 SDT_PROBE_DEFINE1(sdt, kernel, softint, disestablish,
234 "void *"/*sih*/);
235
236 SDT_PROBE_DEFINE2(sdt, kernel, softint, schedule,
237 "void *"/*sih*/,
238 "struct cpu_info *"/*ci*/);
239
240 SDT_PROBE_DEFINE4(sdt, kernel, softint, entry,
241 "void *"/*sih*/,
242 "void (*)(void *)"/*func*/,
243 "void *"/*arg*/,
244 "unsigned"/*flags*/);
245
246 SDT_PROBE_DEFINE4(sdt, kernel, softint, return,
247 "void *"/*sih*/,
248 "void (*)(void *)"/*func*/,
249 "void *"/*arg*/,
250 "unsigned"/*flags*/);
251
252 /*
253 * softint_init_isr:
254 *
255 * Initialize a single interrupt level for a single CPU.
256 */
257 static void
softint_init_isr(softcpu_t * sc,const char * desc,pri_t pri,u_int level,int ipl)258 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level,
259 int ipl)
260 {
261 struct cpu_info *ci;
262 softint_t *si;
263 int error;
264
265 si = &sc->sc_int[level];
266 ci = sc->sc_cpu;
267 si->si_cpu = ci;
268
269 SIMPLEQ_INIT(&si->si_q);
270
271 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR |
272 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp,
273 "soft%s/%u", desc, ci->ci_index);
274 if (error != 0)
275 panic("softint_init_isr: error %d", error);
276
277 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc,
278 ci->ci_index);
279 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL,
280 "softint", si->si_name);
281 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u",
282 desc, ci->ci_index);
283 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL,
284 "softint", si->si_name_block);
285
286 si->si_ipl = ipl;
287 si->si_lwp->l_private = si;
288 softint_init_md(si->si_lwp, level, &si->si_machdep);
289 }
290
291 /*
292 * softint_init:
293 *
294 * Initialize per-CPU data structures. Called from mi_cpu_attach().
295 */
296 void
softint_init(struct cpu_info * ci)297 softint_init(struct cpu_info *ci)
298 {
299 static struct cpu_info *first;
300 softcpu_t *sc, *scfirst;
301 softhand_t *sh, *shmax;
302
303 if (first == NULL) {
304 /* Boot CPU. */
305 first = ci;
306 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE);
307 softint_bytes = round_page(softint_bytes);
308 softint_max = (softint_bytes - sizeof(softcpu_t)) /
309 sizeof(softhand_t);
310 }
311
312 /* Use uvm_km(9) for persistent, page-aligned allocation. */
313 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0,
314 UVM_KMF_WIRED | UVM_KMF_ZERO);
315 if (sc == NULL)
316 panic("softint_init_cpu: cannot allocate memory");
317
318 ci->ci_data.cpu_softcpu = sc;
319 ci->ci_data.cpu_softints = 0;
320 sc->sc_cpu = ci;
321
322 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET,
323 IPL_SOFTNET);
324 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO,
325 IPL_SOFTBIO);
326 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK,
327 IPL_SOFTCLOCK);
328 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL,
329 IPL_SOFTSERIAL);
330
331 if (first != ci) {
332 mutex_enter(&softint_lock);
333 scfirst = first->ci_data.cpu_softcpu;
334 sh = sc->sc_hand;
335 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max);
336 /* Update pointers for this CPU. */
337 for (shmax = sh + softint_max; sh < shmax; sh++) {
338 if (sh->sh_func == NULL)
339 continue;
340 sh->sh_isr =
341 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK];
342 }
343 mutex_exit(&softint_lock);
344 }
345 }
346
347 /*
348 * softint_establish:
349 *
350 * Register a software interrupt handler.
351 */
352 void *
softint_establish(u_int flags,void (* func)(void *),void * arg)353 softint_establish(u_int flags, void (*func)(void *), void *arg)
354 {
355 CPU_INFO_ITERATOR cii;
356 struct cpu_info *ci;
357 softcpu_t *sc;
358 softhand_t *sh;
359 u_int level, index;
360 u_int ipi_id = 0;
361 void *sih;
362
363 level = (flags & SOFTINT_LVLMASK);
364 KASSERT(level < SOFTINT_COUNT);
365 KASSERT((flags & SOFTINT_IMPMASK) == 0);
366
367 mutex_enter(&softint_lock);
368
369 /* Find a free slot. */
370 sc = curcpu()->ci_data.cpu_softcpu;
371 for (index = 1; index < softint_max; index++) {
372 if (sc->sc_hand[index].sh_func == NULL)
373 break;
374 }
375 if (index == softint_max) {
376 mutex_exit(&softint_lock);
377 printf("WARNING: softint_establish: table full, "
378 "increase softint_bytes\n");
379 return NULL;
380 }
381 sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc);
382
383 if (flags & SOFTINT_RCPU) {
384 if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) {
385 mutex_exit(&softint_lock);
386 return NULL;
387 }
388 }
389
390 /* Set up the handler on each CPU. */
391 if (ncpu < 2) {
392 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */
393 sc = curcpu()->ci_data.cpu_softcpu;
394 sh = &sc->sc_hand[index];
395 sh->sh_isr = &sc->sc_int[level];
396 sh->sh_func = func;
397 sh->sh_arg = arg;
398 sh->sh_flags = flags;
399 sh->sh_ipi_id = ipi_id;
400 } else for (CPU_INFO_FOREACH(cii, ci)) {
401 sc = ci->ci_data.cpu_softcpu;
402 sh = &sc->sc_hand[index];
403 sh->sh_isr = &sc->sc_int[level];
404 sh->sh_func = func;
405 sh->sh_arg = arg;
406 sh->sh_flags = flags;
407 sh->sh_ipi_id = ipi_id;
408 }
409 mutex_exit(&softint_lock);
410
411 SDT_PROBE4(sdt, kernel, softint, establish, sih, func, arg, flags);
412
413 return sih;
414 }
415
416 /*
417 * softint_disestablish:
418 *
419 * Unregister a software interrupt handler. The soft interrupt could
420 * still be active at this point, but the caller commits not to try
421 * and trigger it again once this call is made. The caller must not
422 * hold any locks that could be taken from soft interrupt context,
423 * because we will wait for the softint to complete if it's still
424 * running.
425 */
426 void
softint_disestablish(void * arg)427 softint_disestablish(void *arg)
428 {
429 CPU_INFO_ITERATOR cii;
430 struct cpu_info *ci;
431 softcpu_t *sc;
432 softhand_t *sh;
433 uintptr_t offset;
434
435 offset = (uintptr_t)arg;
436 KASSERT(offset != 0);
437 KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
438 offset, softint_bytes);
439
440 /*
441 * Unregister IPI handler if there is any. Note: there is no need
442 * to disable preemption here - ID is stable.
443 */
444 sc = curcpu()->ci_data.cpu_softcpu;
445 sh = (softhand_t *)((uint8_t *)sc + offset);
446 if (sh->sh_ipi_id) {
447 ipi_unregister(sh->sh_ipi_id);
448 }
449
450 /*
451 * Run a dummy softint at the same level on all CPUs and wait for
452 * completion, to make sure this softint is no longer running
453 * anywhere.
454 */
455 xc_barrier(XC_HIGHPRI_IPL(sh->sh_isr->si_ipl));
456
457 /*
458 * Notify dtrace probe when the old softint can't be running
459 * any more, but before it can be recycled for a new softint.
460 */
461 SDT_PROBE1(sdt, kernel, softint, disestablish, arg);
462
463 /* Clear the handler on each CPU. */
464 mutex_enter(&softint_lock);
465 for (CPU_INFO_FOREACH(cii, ci)) {
466 sc = ci->ci_data.cpu_softcpu;
467 sh = (softhand_t *)((uint8_t *)sc + offset);
468 KASSERT(sh->sh_func != NULL);
469 sh->sh_func = NULL;
470 }
471 mutex_exit(&softint_lock);
472 }
473
474 /*
475 * softint_schedule:
476 *
477 * Trigger a software interrupt. Must be called from a hardware
478 * interrupt handler, or with preemption disabled (since we are
479 * using the value of curcpu()).
480 */
481 void
softint_schedule(void * arg)482 softint_schedule(void *arg)
483 {
484 softhand_t *sh;
485 softint_t *si;
486 uintptr_t offset;
487 int s;
488
489 SDT_PROBE2(sdt, kernel, softint, schedule, arg, /*ci*/NULL);
490
491 /*
492 * If this assert fires, rather than disabling preemption explicitly
493 * to make it stop, consider that you are probably using a softint
494 * when you don't need to.
495 */
496 KASSERT(kpreempt_disabled());
497
498 /* Find the handler record for this CPU. */
499 offset = (uintptr_t)arg;
500 KASSERT(offset != 0);
501 KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
502 offset, softint_bytes);
503 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset);
504
505 /* If it's already pending there's nothing to do. */
506 if ((sh->sh_flags & SOFTINT_PENDING) != 0) {
507 return;
508 }
509
510 /*
511 * Enqueue the handler into the LWP's pending list.
512 * If the LWP is completely idle, then make it run.
513 */
514 s = splhigh();
515 if ((sh->sh_flags & SOFTINT_PENDING) == 0) {
516 si = sh->sh_isr;
517 sh->sh_flags |= SOFTINT_PENDING;
518 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q);
519 if (si->si_active == 0) {
520 si->si_active = 1;
521 softint_trigger(si->si_machdep);
522 }
523 }
524 splx(s);
525 }
526
527 /*
528 * softint_schedule_cpu:
529 *
530 * Trigger a software interrupt on a target CPU. This invokes
531 * softint_schedule() for the local CPU or send an IPI to invoke
532 * this routine on the remote CPU. Preemption must be disabled.
533 */
534 void
softint_schedule_cpu(void * arg,struct cpu_info * ci)535 softint_schedule_cpu(void *arg, struct cpu_info *ci)
536 {
537 KASSERT(kpreempt_disabled());
538
539 if (curcpu() != ci) {
540 const softcpu_t *sc = ci->ci_data.cpu_softcpu;
541 const uintptr_t offset = (uintptr_t)arg;
542 const softhand_t *sh;
543
544 SDT_PROBE2(sdt, kernel, softint, schedule, arg, ci);
545 sh = (const softhand_t *)((const uint8_t *)sc + offset);
546 KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0);
547 ipi_trigger(sh->sh_ipi_id, ci);
548 return;
549 }
550
551 /* Just a local CPU. */
552 softint_schedule(arg);
553 }
554
555 /*
556 * softint_execute:
557 *
558 * Invoke handlers for the specified soft interrupt.
559 * Must be entered at splhigh. Will drop the priority
560 * to the level specified, but returns back at splhigh.
561 */
562 static inline void
softint_execute(lwp_t * l,int s)563 softint_execute(lwp_t *l, int s)
564 {
565 softint_t *si = l->l_private;
566 softhand_t *sh;
567
568 KASSERT(si->si_lwp == curlwp);
569 KASSERT(si->si_cpu == curcpu());
570 KASSERT(si->si_lwp->l_wchan == NULL);
571 KASSERT(si->si_active);
572 KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d",
573 l, l->l_nopreempt);
574
575 /*
576 * Note: due to priority inheritance we may have interrupted a
577 * higher priority LWP. Since the soft interrupt must be quick
578 * and is non-preemptable, we don't bother yielding.
579 */
580
581 while (!SIMPLEQ_EMPTY(&si->si_q)) {
582 /*
583 * Pick the longest waiting handler to run. We block
584 * interrupts but do not lock in order to do this, as
585 * we are protecting against the local CPU only.
586 */
587 sh = SIMPLEQ_FIRST(&si->si_q);
588 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q);
589 KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0);
590 sh->sh_flags ^= SOFTINT_PENDING;
591 splx(s);
592
593 /* Run the handler. */
594 SDT_PROBE4(sdt, kernel, softint, entry,
595 ((const char *)sh -
596 (const char *)curcpu()->ci_data.cpu_softcpu),
597 sh->sh_func, sh->sh_arg, sh->sh_flags);
598 if (__predict_true((sh->sh_flags & SOFTINT_MPSAFE) != 0)) {
599 (*sh->sh_func)(sh->sh_arg);
600 } else {
601 KERNEL_LOCK(1, l);
602 (*sh->sh_func)(sh->sh_arg);
603 KERNEL_UNLOCK_ONE(l);
604 }
605 SDT_PROBE4(sdt, kernel, softint, return,
606 ((const char *)sh -
607 (const char *)curcpu()->ci_data.cpu_softcpu),
608 sh->sh_func, sh->sh_arg, sh->sh_flags);
609
610 /* Diagnostic: check that spin-locks have not leaked. */
611 KASSERTMSG(curcpu()->ci_mtx_count == 0,
612 "%s: ci_mtx_count (%d) != 0, sh_func %p\n",
613 __func__, curcpu()->ci_mtx_count, sh->sh_func);
614 /* Diagnostic: check that psrefs have not leaked. */
615 KASSERTMSG(l->l_psrefs == 0, "%s: l_psrefs=%d, sh_func=%p\n",
616 __func__, l->l_psrefs, sh->sh_func);
617 /* Diagnostic: check that biglocks have not leaked. */
618 KASSERTMSG(l->l_blcnt == 0,
619 "%s: sh_func=%p leaked %d biglocks",
620 __func__, sh->sh_func, curlwp->l_blcnt);
621 /* Diagnostic: check that LWP nopreempt remains zero. */
622 KASSERTMSG(l->l_nopreempt == 0,
623 "%s: lwp %p nopreempt %d func %p",
624 __func__, l, l->l_nopreempt, sh->sh_func);
625
626 (void)splhigh();
627 }
628
629 PSREF_DEBUG_BARRIER();
630
631 CPU_COUNT(CPU_COUNT_NSOFT, 1);
632
633 KASSERT(si->si_cpu == curcpu());
634 KASSERT(si->si_lwp->l_wchan == NULL);
635 KASSERT(si->si_active);
636 si->si_evcnt.ev_count++;
637 si->si_active = 0;
638 }
639
640 /*
641 * softint_block:
642 *
643 * Update statistics when the soft interrupt blocks.
644 */
645 void
softint_block(lwp_t * l)646 softint_block(lwp_t *l)
647 {
648 softint_t *si = l->l_private;
649
650 KASSERT((l->l_pflag & LP_INTR) != 0);
651 si->si_evcnt_block.ev_count++;
652 }
653
654 #ifndef __HAVE_FAST_SOFTINTS
655
656 #ifdef __HAVE_PREEMPTION
657 #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS
658 #endif
659
660 /*
661 * softint_init_md:
662 *
663 * Slow path: perform machine-dependent initialization.
664 */
665 void
softint_init_md(lwp_t * l,u_int level,uintptr_t * machdep)666 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
667 {
668 struct proc *p;
669 softint_t *si;
670
671 *machdep = (1 << level);
672 si = l->l_private;
673 p = l->l_proc;
674
675 mutex_enter(p->p_lock);
676 lwp_lock(l);
677 /* Cheat and make the KASSERT in softint_thread() happy. */
678 si->si_active = 1;
679 setrunnable(l);
680 /* LWP now unlocked */
681 mutex_exit(p->p_lock);
682 }
683
684 /*
685 * softint_trigger:
686 *
687 * Slow path: cause a soft interrupt handler to begin executing.
688 * Called at IPL_HIGH.
689 */
690 void
softint_trigger(uintptr_t machdep)691 softint_trigger(uintptr_t machdep)
692 {
693 struct cpu_info *ci;
694 lwp_t *l;
695
696 ci = curcpu();
697 ci->ci_data.cpu_softints |= machdep;
698 l = ci->ci_onproc;
699
700 /*
701 * Arrange for mi_switch() to be called. If called from interrupt
702 * mode, we don't know if curlwp is executing in kernel or user, so
703 * post an AST and have it take a trip through userret(). If not in
704 * interrupt mode, curlwp is running in kernel and will notice the
705 * resched soon enough; avoid the AST.
706 */
707 if (l == ci->ci_data.cpu_idlelwp) {
708 atomic_or_uint(&ci->ci_want_resched,
709 RESCHED_IDLE | RESCHED_UPREEMPT);
710 } else {
711 atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT);
712 if (cpu_intr_p()) {
713 cpu_signotify(l);
714 }
715 }
716 }
717
718 /*
719 * softint_thread:
720 *
721 * Slow path: MI software interrupt dispatch.
722 */
723 void
softint_thread(void * cookie)724 softint_thread(void *cookie)
725 {
726 softint_t *si;
727 lwp_t *l;
728 int s;
729
730 l = curlwp;
731 si = l->l_private;
732
733 for (;;) {
734 /* Clear pending status and run it. */
735 s = splhigh();
736 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep;
737 softint_execute(l, s);
738 splx(s);
739
740 /* Interrupts allowed to run again before switching. */
741 lwp_lock(l);
742 l->l_stat = LSIDL;
743 spc_lock(l->l_cpu);
744 mi_switch(l);
745 }
746 }
747
748 /*
749 * softint_picklwp:
750 *
751 * Slow path: called from mi_switch() to pick the highest priority
752 * soft interrupt LWP that needs to run.
753 */
754 lwp_t *
softint_picklwp(void)755 softint_picklwp(void)
756 {
757 struct cpu_info *ci;
758 u_int mask;
759 softint_t *si;
760 lwp_t *l;
761
762 ci = curcpu();
763 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
764 mask = ci->ci_data.cpu_softints;
765
766 if ((mask & (1 << SOFTINT_SERIAL)) != 0) {
767 l = si[SOFTINT_SERIAL].si_lwp;
768 } else if ((mask & (1 << SOFTINT_NET)) != 0) {
769 l = si[SOFTINT_NET].si_lwp;
770 } else if ((mask & (1 << SOFTINT_BIO)) != 0) {
771 l = si[SOFTINT_BIO].si_lwp;
772 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) {
773 l = si[SOFTINT_CLOCK].si_lwp;
774 } else {
775 panic("softint_picklwp");
776 }
777
778 return l;
779 }
780
781 #else /* !__HAVE_FAST_SOFTINTS */
782
783 /*
784 * softint_thread:
785 *
786 * Fast path: the LWP is switched to without restoring any state,
787 * so we should not arrive here - there is a direct handoff between
788 * the interrupt stub and softint_dispatch().
789 */
790 void
softint_thread(void * cookie)791 softint_thread(void *cookie)
792 {
793
794 panic("softint_thread");
795 }
796
797 /*
798 * softint_dispatch:
799 *
800 * Fast path: entry point from machine-dependent code.
801 */
802 void
softint_dispatch(lwp_t * pinned,int s)803 softint_dispatch(lwp_t *pinned, int s)
804 {
805 struct bintime now;
806 u_int timing;
807 lwp_t *l;
808
809 #ifdef DIAGNOSTIC
810 if ((pinned->l_pflag & LP_RUNNING) == 0 || curlwp->l_stat != LSIDL) {
811 struct lwp *onproc = curcpu()->ci_onproc;
812 int s2 = splhigh();
813 printf("curcpu=%d, spl=%d curspl=%d\n"
814 "onproc=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
815 "curlwp=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
816 "pinned=%p => l_stat=%d l_flag=%08x l_cpu=%d\n",
817 cpu_index(curcpu()), s, s2, onproc, onproc->l_stat,
818 onproc->l_flag, cpu_index(onproc->l_cpu), curlwp,
819 curlwp->l_stat, curlwp->l_flag,
820 cpu_index(curlwp->l_cpu), pinned, pinned->l_stat,
821 pinned->l_flag, cpu_index(pinned->l_cpu));
822 splx(s2);
823 panic("softint screwup");
824 }
825 #endif
826
827 /*
828 * Note the interrupted LWP, and mark the current LWP as running
829 * before proceeding. Although this must as a rule be done with
830 * the LWP locked, at this point no external agents will want to
831 * modify the interrupt LWP's state.
832 */
833 timing = softint_timing;
834 l = curlwp;
835 l->l_switchto = pinned;
836 l->l_stat = LSONPROC;
837
838 /*
839 * Dispatch the interrupt. If softints are being timed, charge
840 * for it.
841 */
842 if (timing) {
843 binuptime(&l->l_stime);
844 membar_producer(); /* for calcru */
845 l->l_pflag |= LP_TIMEINTR;
846 }
847 l->l_pflag |= LP_RUNNING;
848 softint_execute(l, s);
849 if (timing) {
850 binuptime(&now);
851 updatertime(l, &now);
852 l->l_pflag &= ~LP_TIMEINTR;
853 }
854
855 /*
856 * If we blocked while handling the interrupt, the pinned LWP is
857 * gone and we are now running as a kthread, so find another LWP to
858 * run. softint_dispatch() won't be reentered until the priority is
859 * finally dropped to IPL_NONE on entry to the next LWP on this CPU.
860 */
861 l->l_stat = LSIDL;
862 if (l->l_switchto == NULL) {
863 lwp_lock(l);
864 spc_lock(l->l_cpu);
865 mi_switch(l);
866 /* NOTREACHED */
867 }
868 l->l_switchto = NULL;
869 l->l_pflag &= ~LP_RUNNING;
870 }
871
872 #endif /* !__HAVE_FAST_SOFTINTS */
873