1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1991 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * William Jolitz.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 /*-
35 * Copyright (c) 2001 Jake Burkholder.
36 * All rights reserved.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
48 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
49 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
50 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
51 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
52 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
53 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
55 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57 * SUCH DAMAGE.
58 *
59 * from: @(#)isa.c 7.2 (Berkeley) 5/13/91
60 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20
61 */
62
63 #include <sys/cdefs.h>
64 __FBSDID("$FreeBSD: stable/12/sys/sparc64/sparc64/intr_machdep.c 336639 2018-07-23 15:36:55Z avg $");
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/bus.h>
69 #include <sys/errno.h>
70 #include <sys/interrupt.h>
71 #include <sys/kernel.h>
72 #include <sys/lock.h>
73 #include <sys/mutex.h>
74 #include <sys/pcpu.h>
75 #include <sys/proc.h>
76 #include <sys/smp.h>
77 #include <sys/sx.h>
78 #include <sys/vmmeter.h>
79
80 #include <machine/frame.h>
81 #include <machine/intr_machdep.h>
82
83 #define MAX_STRAY_LOG 5
84
85 CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector));
86
87 ih_func_t *intr_handlers[PIL_MAX];
88 uint16_t pil_countp[PIL_MAX];
89 static uint16_t pil_stray_count[PIL_MAX];
90
91 struct intr_vector intr_vectors[IV_MAX];
92 uint16_t intr_countp[IV_MAX];
93 static uint16_t intr_stray_count[IV_MAX];
94
95 static const char *const pil_names[] = {
96 "stray",
97 "low", /* PIL_LOW */
98 "preempt", /* PIL_PREEMPT */
99 "ithrd", /* PIL_ITHREAD */
100 "rndzvs", /* PIL_RENDEZVOUS */
101 "ast", /* PIL_AST */
102 "hardclock", /* PIL_HARDCLOCK */
103 "stray", "stray", "stray", "stray",
104 "filter", /* PIL_FILTER */
105 "bridge", /* PIL_BRIDGE */
106 "stop", /* PIL_STOP */
107 "tick", /* PIL_TICK */
108 };
109
110 /* protect the intr_vectors table */
111 static struct sx intr_table_lock;
112 /* protect intrcnt_index */
113 static struct mtx intrcnt_lock;
114
115 #ifdef SMP
116 static int assign_cpu;
117
118 static void intr_assign_next_cpu(struct intr_vector *iv);
119 static void intr_shuffle_irqs(void *arg __unused);
120 #endif
121
122 static int intr_assign_cpu(void *arg, int cpu);
123 static void intr_execute_handlers(void *);
124 static void intr_stray_level(struct trapframe *);
125 static void intr_stray_vector(void *);
126 static int intrcnt_setname(const char *, int);
127 static void intrcnt_updatename(int, const char *, int);
128 void counter_intr_inc(void);
129
130 static void
intrcnt_updatename(int vec,const char * name,int ispil)131 intrcnt_updatename(int vec, const char *name, int ispil)
132 {
133 static int intrcnt_index, stray_pil_index, stray_vec_index;
134 int name_index;
135
136 mtx_lock_spin(&intrcnt_lock);
137 if (intrnames[0] == '\0') {
138 /* for bitbucket */
139 if (bootverbose)
140 printf("initalizing intr_countp\n");
141 intrcnt_setname("???", intrcnt_index++);
142
143 stray_vec_index = intrcnt_index++;
144 intrcnt_setname("stray", stray_vec_index);
145 for (name_index = 0; name_index < IV_MAX; name_index++)
146 intr_countp[name_index] = stray_vec_index;
147
148 stray_pil_index = intrcnt_index++;
149 intrcnt_setname("pil", stray_pil_index);
150 for (name_index = 0; name_index < PIL_MAX; name_index++)
151 pil_countp[name_index] = stray_pil_index;
152 }
153
154 if (name == NULL)
155 name = "???";
156
157 if (!ispil && intr_countp[vec] != stray_vec_index)
158 name_index = intr_countp[vec];
159 else if (ispil && pil_countp[vec] != stray_pil_index)
160 name_index = pil_countp[vec];
161 else
162 name_index = intrcnt_index++;
163
164 if (intrcnt_setname(name, name_index))
165 name_index = 0;
166
167 if (!ispil)
168 intr_countp[vec] = name_index;
169 else
170 pil_countp[vec] = name_index;
171 mtx_unlock_spin(&intrcnt_lock);
172 }
173
174 static int
intrcnt_setname(const char * name,int index)175 intrcnt_setname(const char *name, int index)
176 {
177
178 if ((MAXCOMLEN + 1) * index >= sintrnames)
179 return (E2BIG);
180 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s",
181 MAXCOMLEN, name);
182 return (0);
183 }
184
185 void
intr_setup(int pri,ih_func_t * ihf,int vec,iv_func_t * ivf,void * iva)186 intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva)
187 {
188 char pilname[MAXCOMLEN + 1];
189 register_t s;
190
191 s = intr_disable();
192 if (vec != -1) {
193 intr_vectors[vec].iv_func = ivf;
194 intr_vectors[vec].iv_arg = iva;
195 intr_vectors[vec].iv_pri = pri;
196 intr_vectors[vec].iv_vec = vec;
197 }
198 intr_handlers[pri] = ihf;
199 intr_restore(s);
200 snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]);
201 intrcnt_updatename(pri, pilname, 1);
202 }
203
204 static void
intr_stray_level(struct trapframe * tf)205 intr_stray_level(struct trapframe *tf)
206 {
207 uint64_t level;
208
209 level = tf->tf_level;
210 if (pil_stray_count[level] < MAX_STRAY_LOG) {
211 printf("stray level interrupt %ld\n", level);
212 pil_stray_count[level]++;
213 if (pil_stray_count[level] >= MAX_STRAY_LOG)
214 printf("got %d stray level interrupt %ld's: not "
215 "logging anymore\n", MAX_STRAY_LOG, level);
216 }
217 }
218
219 static void
intr_stray_vector(void * cookie)220 intr_stray_vector(void *cookie)
221 {
222 struct intr_vector *iv;
223 u_int vec;
224
225 iv = cookie;
226 vec = iv->iv_vec;
227 if (intr_stray_count[vec] < MAX_STRAY_LOG) {
228 printf("stray vector interrupt %d\n", vec);
229 intr_stray_count[vec]++;
230 if (intr_stray_count[vec] >= MAX_STRAY_LOG)
231 printf("got %d stray vector interrupt %d's: not "
232 "logging anymore\n", MAX_STRAY_LOG, vec);
233 }
234 }
235
236 void
intr_init1()237 intr_init1()
238 {
239 int i;
240
241 /* Mark all interrupts as being stray. */
242 for (i = 0; i < PIL_MAX; i++)
243 intr_handlers[i] = intr_stray_level;
244 for (i = 0; i < IV_MAX; i++) {
245 intr_vectors[i].iv_func = intr_stray_vector;
246 intr_vectors[i].iv_arg = &intr_vectors[i];
247 intr_vectors[i].iv_pri = PIL_LOW;
248 intr_vectors[i].iv_vec = i;
249 intr_vectors[i].iv_refcnt = 0;
250 }
251 intr_handlers[PIL_LOW] = intr_fast;
252 }
253
254 void
intr_init2()255 intr_init2()
256 {
257
258 sx_init(&intr_table_lock, "intr sources");
259 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
260 }
261
262 static int
intr_assign_cpu(void * arg,int cpu)263 intr_assign_cpu(void *arg, int cpu)
264 {
265 #ifdef SMP
266 struct pcpu *pc;
267 struct intr_vector *iv;
268
269 /*
270 * Don't do anything during early boot. We will pick up the
271 * assignment once the APs are started.
272 */
273 if (assign_cpu && cpu != NOCPU) {
274 pc = pcpu_find(cpu);
275 if (pc == NULL)
276 return (EINVAL);
277 iv = arg;
278 sx_xlock(&intr_table_lock);
279 iv->iv_mid = pc->pc_mid;
280 iv->iv_ic->ic_assign(iv);
281 sx_xunlock(&intr_table_lock);
282 }
283 return (0);
284 #else
285 return (EOPNOTSUPP);
286 #endif
287 }
288
289 static void
intr_execute_handlers(void * cookie)290 intr_execute_handlers(void *cookie)
291 {
292 struct intr_vector *iv;
293
294 iv = cookie;
295 if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0))
296 intr_stray_vector(iv);
297 }
298
299 int
intr_controller_register(int vec,const struct intr_controller * ic,void * icarg)300 intr_controller_register(int vec, const struct intr_controller *ic,
301 void *icarg)
302 {
303 struct intr_event *ie;
304 struct intr_vector *iv;
305 int error;
306
307 if (vec < 0 || vec >= IV_MAX)
308 return (EINVAL);
309 sx_xlock(&intr_table_lock);
310 iv = &intr_vectors[vec];
311 ie = iv->iv_event;
312 sx_xunlock(&intr_table_lock);
313 if (ie != NULL)
314 return (EEXIST);
315 error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear,
316 ic->ic_clear, intr_assign_cpu, "vec%d:", vec);
317 if (error != 0)
318 return (error);
319 sx_xlock(&intr_table_lock);
320 if (iv->iv_event != NULL) {
321 sx_xunlock(&intr_table_lock);
322 intr_event_destroy(ie);
323 return (EEXIST);
324 }
325 iv->iv_ic = ic;
326 iv->iv_icarg = icarg;
327 iv->iv_event = ie;
328 iv->iv_mid = PCPU_GET(mid);
329 sx_xunlock(&intr_table_lock);
330 return (0);
331 }
332
333 int
inthand_add(const char * name,int vec,driver_filter_t * filt,driver_intr_t * handler,void * arg,int flags,void ** cookiep)334 inthand_add(const char *name, int vec, driver_filter_t *filt,
335 driver_intr_t *handler, void *arg, int flags, void **cookiep)
336 {
337 const struct intr_controller *ic;
338 struct intr_event *ie;
339 struct intr_handler *ih;
340 struct intr_vector *iv;
341 int error, filter;
342
343 if (vec < 0 || vec >= IV_MAX)
344 return (EINVAL);
345 /*
346 * INTR_BRIDGE filters/handlers are special purpose only, allowing
347 * them to be shared just would complicate things unnecessarily.
348 */
349 if ((flags & INTR_BRIDGE) != 0 && (flags & INTR_EXCL) == 0)
350 return (EINVAL);
351 sx_xlock(&intr_table_lock);
352 iv = &intr_vectors[vec];
353 ic = iv->iv_ic;
354 ie = iv->iv_event;
355 sx_xunlock(&intr_table_lock);
356 if (ic == NULL || ie == NULL)
357 return (EINVAL);
358 error = intr_event_add_handler(ie, name, filt, handler, arg,
359 intr_priority(flags), flags, cookiep);
360 if (error != 0)
361 return (error);
362 sx_xlock(&intr_table_lock);
363 /* Disable the interrupt while we fiddle with it. */
364 ic->ic_disable(iv);
365 iv->iv_refcnt++;
366 if (iv->iv_refcnt == 1)
367 intr_setup((flags & INTR_BRIDGE) != 0 ? PIL_BRIDGE :
368 filt != NULL ? PIL_FILTER : PIL_ITHREAD, intr_fast,
369 vec, intr_execute_handlers, iv);
370 else if (filt != NULL) {
371 /*
372 * Check if we need to upgrade from PIL_ITHREAD to PIL_FILTER.
373 * Given that apart from the on-board SCCs and UARTs shared
374 * interrupts are rather uncommon on sparc64 this should be
375 * pretty rare in practice.
376 */
377 filter = 0;
378 CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
379 if (ih->ih_filter != NULL && ih->ih_filter != filt) {
380 filter = 1;
381 break;
382 }
383 }
384 if (filter == 0)
385 intr_setup(PIL_FILTER, intr_fast, vec,
386 intr_execute_handlers, iv);
387 }
388 intr_stray_count[vec] = 0;
389 intrcnt_updatename(vec, ie->ie_fullname, 0);
390 #ifdef SMP
391 if (assign_cpu)
392 intr_assign_next_cpu(iv);
393 #endif
394 ic->ic_enable(iv);
395 /* Ensure the interrupt is cleared, it might have triggered before. */
396 if (ic->ic_clear != NULL)
397 ic->ic_clear(iv);
398 sx_xunlock(&intr_table_lock);
399 return (0);
400 }
401
402 int
inthand_remove(int vec,void * cookie)403 inthand_remove(int vec, void *cookie)
404 {
405 struct intr_vector *iv;
406 int error;
407
408 if (vec < 0 || vec >= IV_MAX)
409 return (EINVAL);
410 error = intr_event_remove_handler(cookie);
411 if (error == 0) {
412 /*
413 * XXX: maybe this should be done regardless of whether
414 * intr_event_remove_handler() succeeded?
415 */
416 sx_xlock(&intr_table_lock);
417 iv = &intr_vectors[vec];
418 iv->iv_refcnt--;
419 if (iv->iv_refcnt == 0) {
420 /*
421 * Don't disable the interrupt for now, so that
422 * stray interrupts get detected...
423 */
424 intr_setup(PIL_LOW, intr_fast, vec,
425 intr_stray_vector, iv);
426 }
427 sx_xunlock(&intr_table_lock);
428 }
429 return (error);
430 }
431
432 /* Add a description to an active interrupt handler. */
433 int
intr_describe(int vec,void * ih,const char * descr)434 intr_describe(int vec, void *ih, const char *descr)
435 {
436 struct intr_vector *iv;
437 int error;
438
439 if (vec < 0 || vec >= IV_MAX)
440 return (EINVAL);
441 sx_xlock(&intr_table_lock);
442 iv = &intr_vectors[vec];
443 if (iv == NULL) {
444 sx_xunlock(&intr_table_lock);
445 return (EINVAL);
446 }
447 error = intr_event_describe_handler(iv->iv_event, ih, descr);
448 if (error) {
449 sx_xunlock(&intr_table_lock);
450 return (error);
451 }
452 intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0);
453 sx_xunlock(&intr_table_lock);
454 return (error);
455 }
456
457 /*
458 * Do VM_CNT_INC(intr), being in the interrupt context already. This is
459 * called from assembly.
460 * To avoid counter_enter() and appropriate assertion, unwrap VM_CNT_INC()
461 * and hardcode the actual increment.
462 */
463 void
counter_intr_inc(void)464 counter_intr_inc(void)
465 {
466
467 *(uint64_t *)zpcpu_get(vm_cnt.v_intr) += 1;
468 }
469
470 #ifdef SMP
471 /*
472 * Support for balancing interrupt sources across CPUs. For now we just
473 * allocate CPUs round-robin.
474 */
475
476 static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1);
477 static int current_cpu;
478
479 static void
intr_assign_next_cpu(struct intr_vector * iv)480 intr_assign_next_cpu(struct intr_vector *iv)
481 {
482 struct pcpu *pc;
483
484 sx_assert(&intr_table_lock, SA_XLOCKED);
485
486 /*
487 * Assign this source to a CPU in a round-robin fashion.
488 */
489 pc = pcpu_find(current_cpu);
490 if (pc == NULL)
491 return;
492 iv->iv_mid = pc->pc_mid;
493 iv->iv_ic->ic_assign(iv);
494 do {
495 current_cpu++;
496 if (current_cpu > mp_maxid)
497 current_cpu = 0;
498 } while (!CPU_ISSET(current_cpu, &intr_cpus));
499 }
500
501 /* Attempt to bind the specified IRQ to the specified CPU. */
502 int
intr_bind(int vec,u_char cpu)503 intr_bind(int vec, u_char cpu)
504 {
505 struct intr_vector *iv;
506 int error;
507
508 if (vec < 0 || vec >= IV_MAX)
509 return (EINVAL);
510 sx_xlock(&intr_table_lock);
511 iv = &intr_vectors[vec];
512 if (iv == NULL) {
513 sx_xunlock(&intr_table_lock);
514 return (EINVAL);
515 }
516 error = intr_event_bind(iv->iv_event, cpu);
517 sx_xunlock(&intr_table_lock);
518 return (error);
519 }
520
521 /*
522 * Add a CPU to our mask of valid CPUs that can be destinations of
523 * interrupts.
524 */
525 void
intr_add_cpu(u_int cpu)526 intr_add_cpu(u_int cpu)
527 {
528
529 if (cpu >= MAXCPU)
530 panic("%s: Invalid CPU ID", __func__);
531 if (bootverbose)
532 printf("INTR: Adding CPU %d as a target\n", cpu);
533
534 CPU_SET(cpu, &intr_cpus);
535 }
536
537 /*
538 * Distribute all the interrupt sources among the available CPUs once the
539 * APs have been launched.
540 */
541 static void
intr_shuffle_irqs(void * arg __unused)542 intr_shuffle_irqs(void *arg __unused)
543 {
544 struct pcpu *pc;
545 struct intr_vector *iv;
546 int i;
547
548 /* Don't bother on UP. */
549 if (mp_ncpus == 1)
550 return;
551
552 sx_xlock(&intr_table_lock);
553 assign_cpu = 1;
554 for (i = 0; i < IV_MAX; i++) {
555 iv = &intr_vectors[i];
556 if (iv != NULL && iv->iv_refcnt > 0) {
557 /*
558 * If this event is already bound to a CPU,
559 * then assign the source to that CPU instead
560 * of picking one via round-robin.
561 */
562 if (iv->iv_event->ie_cpu != NOCPU &&
563 (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) {
564 iv->iv_mid = pc->pc_mid;
565 iv->iv_ic->ic_assign(iv);
566 } else
567 intr_assign_next_cpu(iv);
568 }
569 }
570 sx_xunlock(&intr_table_lock);
571 }
572 SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs,
573 NULL);
574 #endif
575