1 /******************************************************************************
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include "opt_ddb.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/bus.h>
40 #include <sys/malloc.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/interrupt.h>
46 #include <sys/pcpu.h>
47 #include <sys/smp.h>
48
49 #include <vm/vm.h>
50 #include <vm/pmap.h>
51
52 #include <machine/intr_machdep.h>
53 #include <x86/apicvar.h>
54 #include <x86/apicreg.h>
55 #include <machine/smp.h>
56 #include <machine/stdarg.h>
57
58 #include <machine/xen/synch_bitops.h>
59 #include <machine/xen/xen-os.h>
60
61 #include <xen/hypervisor.h>
62 #include <xen/xen_intr.h>
63 #include <xen/evtchn/evtchnvar.h>
64
65 #include <dev/xen/xenpci/xenpcivar.h>
66 #include <dev/pci/pcivar.h>
67
68 #ifdef DDB
69 #include <ddb/ddb.h>
70 #endif
71
72 static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
73
74 /**
75 * Per-cpu event channel processing state.
76 */
77 struct xen_intr_pcpu_data {
78 /**
79 * The last event channel bitmap section (level one bit) processed.
80 * This is used to ensure we scan all ports before
81 * servicing an already servied port again.
82 */
83 u_int last_processed_l1i;
84
85 /**
86 * The last event channel processed within the event channel
87 * bitmap being scanned.
88 */
89 u_int last_processed_l2i;
90
91 /** Pointer to this CPU's interrupt statistic counter. */
92 u_long *evtchn_intrcnt;
93
94 /**
95 * A bitmap of ports that can be serviced from this CPU.
96 * A set bit means interrupt handling is enabled.
97 */
98 u_long evtchn_enabled[sizeof(u_long) * 8];
99 };
100
101 /*
102 * Start the scan at port 0 by initializing the last scanned
103 * location as the highest numbered event channel port.
104 */
105 static DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
106 .last_processed_l1i = LONG_BIT - 1,
107 .last_processed_l2i = LONG_BIT - 1
108 };
109
110 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
111
112 #define XEN_EEXIST 17 /* Xen "already exists" error */
113 #define XEN_ALLOCATE_VECTOR 0 /* Allocate a vector for this event channel */
114 #define XEN_INVALID_EVTCHN 0 /* Invalid event channel */
115
116 #define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN)
117
118 struct xenisrc {
119 struct intsrc xi_intsrc;
120 enum evtchn_type xi_type;
121 int xi_cpu; /* VCPU for delivery. */
122 int xi_vector; /* Global isrc vector number. */
123 evtchn_port_t xi_port;
124 int xi_pirq;
125 int xi_virq;
126 void *xi_cookie;
127 u_int xi_close:1; /* close on unbind? */
128 u_int xi_activehi:1;
129 u_int xi_edgetrigger:1;
130 u_int xi_masked:1;
131 };
132
133 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
134
135 static void xen_intr_suspend(struct pic *);
136 static void xen_intr_resume(struct pic *, bool suspend_cancelled);
137 static void xen_intr_enable_source(struct intsrc *isrc);
138 static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
139 static void xen_intr_eoi_source(struct intsrc *isrc);
140 static void xen_intr_enable_intr(struct intsrc *isrc);
141 static void xen_intr_disable_intr(struct intsrc *isrc);
142 static int xen_intr_vector(struct intsrc *isrc);
143 static int xen_intr_source_pending(struct intsrc *isrc);
144 static int xen_intr_config_intr(struct intsrc *isrc,
145 enum intr_trigger trig, enum intr_polarity pol);
146 static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
147
148 static void xen_intr_pirq_enable_source(struct intsrc *isrc);
149 static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi);
150 static void xen_intr_pirq_eoi_source(struct intsrc *isrc);
151 static void xen_intr_pirq_enable_intr(struct intsrc *isrc);
152 static void xen_intr_pirq_disable_intr(struct intsrc *isrc);
153 static int xen_intr_pirq_config_intr(struct intsrc *isrc,
154 enum intr_trigger trig, enum intr_polarity pol);
155
156 /**
157 * PIC interface for all event channel port types except physical IRQs.
158 */
159 struct pic xen_intr_pic = {
160 .pic_enable_source = xen_intr_enable_source,
161 .pic_disable_source = xen_intr_disable_source,
162 .pic_eoi_source = xen_intr_eoi_source,
163 .pic_enable_intr = xen_intr_enable_intr,
164 .pic_disable_intr = xen_intr_disable_intr,
165 .pic_vector = xen_intr_vector,
166 .pic_source_pending = xen_intr_source_pending,
167 .pic_suspend = xen_intr_suspend,
168 .pic_resume = xen_intr_resume,
169 .pic_config_intr = xen_intr_config_intr,
170 .pic_assign_cpu = xen_intr_assign_cpu
171 };
172
173 /**
174 * PIC interface for all event channel representing
175 * physical interrupt sources.
176 */
177 struct pic xen_intr_pirq_pic = {
178 .pic_enable_source = xen_intr_pirq_enable_source,
179 .pic_disable_source = xen_intr_pirq_disable_source,
180 .pic_eoi_source = xen_intr_pirq_eoi_source,
181 .pic_enable_intr = xen_intr_pirq_enable_intr,
182 .pic_disable_intr = xen_intr_pirq_disable_intr,
183 .pic_vector = xen_intr_vector,
184 .pic_source_pending = xen_intr_source_pending,
185 .pic_config_intr = xen_intr_pirq_config_intr,
186 .pic_assign_cpu = xen_intr_assign_cpu
187 };
188
189 static struct mtx xen_intr_isrc_lock;
190 static int xen_intr_auto_vector_count;
191 static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
192 static u_long *xen_intr_pirq_eoi_map;
193 static boolean_t xen_intr_pirq_eoi_map_enabled;
194
195 /*------------------------- Private Functions --------------------------------*/
196 /**
197 * Disable signal delivery for an event channel port on the
198 * specified CPU.
199 *
200 * \param port The event channel port to mask.
201 *
202 * This API is used to manage the port<=>CPU binding of event
203 * channel handlers.
204 *
205 * \note This operation does not preclude reception of an event
206 * for this event channel on another CPU. To mask the
207 * event channel globally, use evtchn_mask().
208 */
209 static inline void
evtchn_cpu_mask_port(u_int cpu,evtchn_port_t port)210 evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
211 {
212 struct xen_intr_pcpu_data *pcpu;
213
214 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
215 xen_clear_bit(port, pcpu->evtchn_enabled);
216 }
217
218 /**
219 * Enable signal delivery for an event channel port on the
220 * specified CPU.
221 *
222 * \param port The event channel port to unmask.
223 *
224 * This API is used to manage the port<=>CPU binding of event
225 * channel handlers.
226 *
227 * \note This operation does not guarantee that event delivery
228 * is enabled for this event channel port. The port must
229 * also be globally enabled. See evtchn_unmask().
230 */
231 static inline void
evtchn_cpu_unmask_port(u_int cpu,evtchn_port_t port)232 evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
233 {
234 struct xen_intr_pcpu_data *pcpu;
235
236 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
237 xen_set_bit(port, pcpu->evtchn_enabled);
238 }
239
240 /**
241 * Allocate and register a per-cpu Xen upcall interrupt counter.
242 *
243 * \param cpu The cpu for which to register this interrupt count.
244 */
245 static void
xen_intr_intrcnt_add(u_int cpu)246 xen_intr_intrcnt_add(u_int cpu)
247 {
248 char buf[MAXCOMLEN + 1];
249 struct xen_intr_pcpu_data *pcpu;
250
251 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
252 if (pcpu->evtchn_intrcnt != NULL)
253 return;
254
255 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
256 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
257 }
258
259 /**
260 * Search for an already allocated but currently unused Xen interrupt
261 * source object.
262 *
263 * \param type Restrict the search to interrupt sources of the given
264 * type.
265 *
266 * \return A pointer to a free Xen interrupt source object or NULL.
267 */
268 static struct xenisrc *
xen_intr_find_unused_isrc(enum evtchn_type type)269 xen_intr_find_unused_isrc(enum evtchn_type type)
270 {
271 int isrc_idx;
272
273 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
274
275 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) {
276 struct xenisrc *isrc;
277 u_int vector;
278
279 vector = FIRST_EVTCHN_INT + isrc_idx;
280 isrc = (struct xenisrc *)intr_lookup_source(vector);
281 if (isrc != NULL
282 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
283 KASSERT(isrc->xi_intsrc.is_handlers == 0,
284 ("Free evtchn still has handlers"));
285 isrc->xi_type = type;
286 return (isrc);
287 }
288 }
289 return (NULL);
290 }
291
292 /**
293 * Allocate a Xen interrupt source object.
294 *
295 * \param type The type of interrupt source to create.
296 *
297 * \return A pointer to a newly allocated Xen interrupt source
298 * object or NULL.
299 */
300 static struct xenisrc *
xen_intr_alloc_isrc(enum evtchn_type type,int vector)301 xen_intr_alloc_isrc(enum evtchn_type type, int vector)
302 {
303 static int warned;
304 struct xenisrc *isrc;
305
306 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
307
308 if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) {
309 if (!warned) {
310 warned = 1;
311 printf("xen_intr_alloc: Event channels exhausted.\n");
312 }
313 return (NULL);
314 }
315
316 if (type != EVTCHN_TYPE_PIRQ) {
317 vector = FIRST_EVTCHN_INT + xen_intr_auto_vector_count;
318 xen_intr_auto_vector_count++;
319 }
320
321 KASSERT((intr_lookup_source(vector) == NULL),
322 ("Trying to use an already allocated vector"));
323
324 mtx_unlock(&xen_intr_isrc_lock);
325 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
326 isrc->xi_intsrc.is_pic =
327 (type == EVTCHN_TYPE_PIRQ) ? &xen_intr_pirq_pic : &xen_intr_pic;
328 isrc->xi_vector = vector;
329 isrc->xi_type = type;
330 intr_register_source(&isrc->xi_intsrc);
331 mtx_lock(&xen_intr_isrc_lock);
332
333 return (isrc);
334 }
335
336 /**
337 * Attempt to free an active Xen interrupt source object.
338 *
339 * \param isrc The interrupt source object to release.
340 *
341 * \returns EBUSY if the source is still in use, otherwise 0.
342 */
343 static int
xen_intr_release_isrc(struct xenisrc * isrc)344 xen_intr_release_isrc(struct xenisrc *isrc)
345 {
346
347 mtx_lock(&xen_intr_isrc_lock);
348 if (isrc->xi_intsrc.is_handlers != 0) {
349 mtx_unlock(&xen_intr_isrc_lock);
350 return (EBUSY);
351 }
352 evtchn_mask_port(isrc->xi_port);
353 evtchn_clear_port(isrc->xi_port);
354
355 /* Rebind port to CPU 0. */
356 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
357 evtchn_cpu_unmask_port(0, isrc->xi_port);
358
359 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
360 struct evtchn_close close = { .port = isrc->xi_port };
361 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
362 panic("EVTCHNOP_close failed");
363 }
364
365 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
366 isrc->xi_cpu = 0;
367 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
368 isrc->xi_port = 0;
369 isrc->xi_cookie = NULL;
370 mtx_unlock(&xen_intr_isrc_lock);
371 return (0);
372 }
373
374 /**
375 * Associate an interrupt handler with an already allocated local Xen
376 * event channel port.
377 *
378 * \param isrcp The returned Xen interrupt object associated with
379 * the specified local port.
380 * \param local_port The event channel to bind.
381 * \param type The event channel type of local_port.
382 * \param intr_owner The device making this bind request.
383 * \param filter An interrupt filter handler. Specify NULL
384 * to always dispatch to the ithread handler.
385 * \param handler An interrupt ithread handler. Optional (can
386 * specify NULL) if all necessary event actions
387 * are performed by filter.
388 * \param arg Argument to present to both filter and handler.
389 * \param irqflags Interrupt handler flags. See sys/bus.h.
390 * \param handlep Pointer to an opaque handle used to manage this
391 * registration.
392 *
393 * \returns 0 on success, otherwise an errno.
394 */
395 static int
xen_intr_bind_isrc(struct xenisrc ** isrcp,evtchn_port_t local_port,enum evtchn_type type,device_t intr_owner,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)396 xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
397 enum evtchn_type type, device_t intr_owner, driver_filter_t filter,
398 driver_intr_t handler, void *arg, enum intr_type flags,
399 xen_intr_handle_t *port_handlep)
400 {
401 struct xenisrc *isrc;
402 int error;
403
404 *isrcp = NULL;
405 if (port_handlep == NULL) {
406 device_printf(intr_owner,
407 "xen_intr_bind_isrc: Bad event handle\n");
408 return (EINVAL);
409 }
410
411 mtx_lock(&xen_intr_isrc_lock);
412 isrc = xen_intr_find_unused_isrc(type);
413 if (isrc == NULL) {
414 isrc = xen_intr_alloc_isrc(type, XEN_ALLOCATE_VECTOR);
415 if (isrc == NULL) {
416 mtx_unlock(&xen_intr_isrc_lock);
417 return (ENOSPC);
418 }
419 }
420 isrc->xi_port = local_port;
421 xen_intr_port_to_isrc[local_port] = isrc;
422 mtx_unlock(&xen_intr_isrc_lock);
423
424 /* Assign the opaque handler (the event channel port) */
425 *port_handlep = &isrc->xi_port;
426
427 #ifdef SMP
428 if (type == EVTCHN_TYPE_PORT) {
429 /*
430 * By default all interrupts are assigned to vCPU#0
431 * unless specified otherwise, so shuffle them to balance
432 * the interrupt load.
433 */
434 xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu());
435 }
436 #endif
437
438 if (filter == NULL && handler == NULL) {
439 /*
440 * No filter/handler provided, leave the event channel
441 * masked and without a valid handler, the caller is
442 * in charge of setting that up.
443 */
444 *isrcp = isrc;
445 return (0);
446 }
447
448 error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags,
449 *port_handlep);
450 if (error != 0) {
451 xen_intr_release_isrc(isrc);
452 return (error);
453 }
454 *isrcp = isrc;
455 return (0);
456 }
457
458 /**
459 * Lookup a Xen interrupt source object given an interrupt binding handle.
460 *
461 * \param handle A handle initialized by a previous call to
462 * xen_intr_bind_isrc().
463 *
464 * \returns A pointer to the Xen interrupt source object associated
465 * with the given interrupt handle. NULL if no association
466 * currently exists.
467 */
468 static struct xenisrc *
xen_intr_isrc(xen_intr_handle_t handle)469 xen_intr_isrc(xen_intr_handle_t handle)
470 {
471 evtchn_port_t port;
472
473 if (handle == NULL)
474 return (NULL);
475
476 port = *(evtchn_port_t *)handle;
477 if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS)
478 return (NULL);
479
480 return (xen_intr_port_to_isrc[port]);
481 }
482
483 /**
484 * Determine the event channel ports at the given section of the
485 * event port bitmap which have pending events for the given cpu.
486 *
487 * \param pcpu The Xen interrupt pcpu data for the cpu being querried.
488 * \param sh The Xen shared info area.
489 * \param idx The index of the section of the event channel bitmap to
490 * inspect.
491 *
492 * \returns A u_long with bits set for every event channel with pending
493 * events.
494 */
495 static inline u_long
xen_intr_active_ports(struct xen_intr_pcpu_data * pcpu,shared_info_t * sh,u_int idx)496 xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
497 u_int idx)
498 {
499
500 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
501 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
502 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
503 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
504 return (sh->evtchn_pending[idx]
505 & ~sh->evtchn_mask[idx]
506 & pcpu->evtchn_enabled[idx]);
507 }
508
509 /**
510 * Interrupt handler for processing all Xen event channel events.
511 *
512 * \param trap_frame The trap frame context for the current interrupt.
513 */
514 void
xen_intr_handle_upcall(struct trapframe * trap_frame)515 xen_intr_handle_upcall(struct trapframe *trap_frame)
516 {
517 u_int l1i, l2i, port, cpu;
518 u_long masked_l1, masked_l2;
519 struct xenisrc *isrc;
520 shared_info_t *s;
521 vcpu_info_t *v;
522 struct xen_intr_pcpu_data *pc;
523 u_long l1, l2;
524
525 /*
526 * Disable preemption in order to always check and fire events
527 * on the right vCPU
528 */
529 critical_enter();
530
531 cpu = PCPU_GET(cpuid);
532 pc = DPCPU_PTR(xen_intr_pcpu);
533 s = HYPERVISOR_shared_info;
534 v = DPCPU_GET(vcpu_info);
535
536 if (xen_hvm_domain() && !xen_vector_callback_enabled) {
537 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
538 }
539
540 v->evtchn_upcall_pending = 0;
541
542 #if 0
543 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
544 /* Clear master flag /before/ clearing selector flag. */
545 wmb();
546 #endif
547 #endif
548
549 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
550
551 l1i = pc->last_processed_l1i;
552 l2i = pc->last_processed_l2i;
553 (*pc->evtchn_intrcnt)++;
554
555 while (l1 != 0) {
556
557 l1i = (l1i + 1) % LONG_BIT;
558 masked_l1 = l1 & ((~0UL) << l1i);
559
560 if (masked_l1 == 0) {
561 /*
562 * if we masked out all events, wrap around
563 * to the beginning.
564 */
565 l1i = LONG_BIT - 1;
566 l2i = LONG_BIT - 1;
567 continue;
568 }
569 l1i = ffsl(masked_l1) - 1;
570
571 do {
572 l2 = xen_intr_active_ports(pc, s, l1i);
573
574 l2i = (l2i + 1) % LONG_BIT;
575 masked_l2 = l2 & ((~0UL) << l2i);
576
577 if (masked_l2 == 0) {
578 /* if we masked out all events, move on */
579 l2i = LONG_BIT - 1;
580 break;
581 }
582 l2i = ffsl(masked_l2) - 1;
583
584 /* process port */
585 port = (l1i * LONG_BIT) + l2i;
586 synch_clear_bit(port, &s->evtchn_pending[0]);
587
588 isrc = xen_intr_port_to_isrc[port];
589 if (__predict_false(isrc == NULL))
590 continue;
591
592 /* Make sure we are firing on the right vCPU */
593 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
594 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
595 PCPU_GET(cpuid), isrc->xi_cpu));
596
597 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
598
599 /*
600 * If this is the final port processed,
601 * we'll pick up here+1 next time.
602 */
603 pc->last_processed_l1i = l1i;
604 pc->last_processed_l2i = l2i;
605
606 } while (l2i != LONG_BIT - 1);
607
608 l2 = xen_intr_active_ports(pc, s, l1i);
609 if (l2 == 0) {
610 /*
611 * We handled all ports, so we can clear the
612 * selector bit.
613 */
614 l1 &= ~(1UL << l1i);
615 }
616 }
617 critical_exit();
618 }
619
620 static int
xen_intr_init(void * dummy __unused)621 xen_intr_init(void *dummy __unused)
622 {
623 shared_info_t *s = HYPERVISOR_shared_info;
624 struct xen_intr_pcpu_data *pcpu;
625 struct physdev_pirq_eoi_gmfn eoi_gmfn;
626 int i, rc;
627
628 if (!xen_domain())
629 return (0);
630
631 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
632
633 /*
634 * Register interrupt count manually as we aren't
635 * guaranteed to see a call to xen_intr_assign_cpu()
636 * before our first interrupt. Also set the per-cpu
637 * mask of CPU#0 to enable all, since by default
638 * all event channels are bound to CPU#0.
639 */
640 CPU_FOREACH(i) {
641 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
642 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
643 sizeof(pcpu->evtchn_enabled));
644 xen_intr_intrcnt_add(i);
645 }
646
647 for (i = 0; i < nitems(s->evtchn_mask); i++)
648 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
649
650 /* Try to register PIRQ EOI map */
651 xen_intr_pirq_eoi_map = malloc(PAGE_SIZE, M_XENINTR, M_WAITOK | M_ZERO);
652 eoi_gmfn.gmfn = atop(vtophys(xen_intr_pirq_eoi_map));
653 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
654 if (rc != 0 && bootverbose)
655 printf("Xen interrupts: unable to register PIRQ EOI map\n");
656 else
657 xen_intr_pirq_eoi_map_enabled = true;
658
659 intr_register_pic(&xen_intr_pic);
660 intr_register_pic(&xen_intr_pirq_pic);
661
662 if (bootverbose)
663 printf("Xen interrupt system initialized\n");
664
665 return (0);
666 }
667 SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
668
669 /*--------------------------- Common PIC Functions ---------------------------*/
670 /**
671 * Prepare this PIC for system suspension.
672 */
673 static void
xen_intr_suspend(struct pic * unused)674 xen_intr_suspend(struct pic *unused)
675 {
676 }
677
678 static void
xen_rebind_ipi(struct xenisrc * isrc)679 xen_rebind_ipi(struct xenisrc *isrc)
680 {
681 #ifdef SMP
682 int cpu = isrc->xi_cpu;
683 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
684 int error;
685 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
686
687 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
688 &bind_ipi);
689 if (error != 0)
690 panic("unable to rebind xen IPI: %d", error);
691
692 isrc->xi_port = bind_ipi.port;
693 isrc->xi_cpu = 0;
694 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
695
696 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
697 cpu_apic_ids[cpu]);
698 if (error)
699 panic("unable to bind xen IPI to CPU#%d: %d",
700 cpu, error);
701
702 evtchn_unmask_port(bind_ipi.port);
703 #else
704 panic("Resume IPI event channel on UP");
705 #endif
706 }
707
708 static void
xen_rebind_virq(struct xenisrc * isrc)709 xen_rebind_virq(struct xenisrc *isrc)
710 {
711 int cpu = isrc->xi_cpu;
712 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
713 int error;
714 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
715 .vcpu = vcpu_id };
716
717 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
718 &bind_virq);
719 if (error != 0)
720 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
721
722 isrc->xi_port = bind_virq.port;
723 isrc->xi_cpu = 0;
724 xen_intr_port_to_isrc[bind_virq.port] = isrc;
725
726 #ifdef SMP
727 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
728 cpu_apic_ids[cpu]);
729 if (error)
730 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
731 isrc->xi_virq, cpu, error);
732 #endif
733
734 evtchn_unmask_port(bind_virq.port);
735 }
736
737 /**
738 * Return this PIC to service after being suspended.
739 */
740 static void
xen_intr_resume(struct pic * unused,bool suspend_cancelled)741 xen_intr_resume(struct pic *unused, bool suspend_cancelled)
742 {
743 shared_info_t *s = HYPERVISOR_shared_info;
744 struct xenisrc *isrc;
745 u_int isrc_idx;
746 int i;
747
748 if (suspend_cancelled)
749 return;
750
751 /* Reset the per-CPU masks */
752 CPU_FOREACH(i) {
753 struct xen_intr_pcpu_data *pcpu;
754
755 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
756 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
757 sizeof(pcpu->evtchn_enabled));
758 }
759
760 /* Mask all event channels. */
761 for (i = 0; i < nitems(s->evtchn_mask); i++)
762 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
763
764 /* Remove port -> isrc mappings */
765 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
766
767 /* Free unused isrcs and rebind VIRQs and IPIs */
768 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) {
769 u_int vector;
770
771 vector = FIRST_EVTCHN_INT + isrc_idx;
772 isrc = (struct xenisrc *)intr_lookup_source(vector);
773 if (isrc != NULL) {
774 isrc->xi_port = 0;
775 switch (isrc->xi_type) {
776 case EVTCHN_TYPE_IPI:
777 xen_rebind_ipi(isrc);
778 break;
779 case EVTCHN_TYPE_VIRQ:
780 xen_rebind_virq(isrc);
781 break;
782 default:
783 intr_remove_handler(isrc->xi_cookie);
784 isrc->xi_cpu = 0;
785 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
786 isrc->xi_cookie = NULL;
787 break;
788 }
789 }
790 }
791 }
792
793 /**
794 * Disable a Xen interrupt source.
795 *
796 * \param isrc The interrupt source to disable.
797 */
798 static void
xen_intr_disable_intr(struct intsrc * base_isrc)799 xen_intr_disable_intr(struct intsrc *base_isrc)
800 {
801 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
802
803 evtchn_mask_port(isrc->xi_port);
804 }
805
806 /**
807 * Determine the global interrupt vector number for
808 * a Xen interrupt source.
809 *
810 * \param isrc The interrupt source to query.
811 *
812 * \return The vector number corresponding to the given interrupt source.
813 */
814 static int
xen_intr_vector(struct intsrc * base_isrc)815 xen_intr_vector(struct intsrc *base_isrc)
816 {
817 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
818
819 return (isrc->xi_vector);
820 }
821
822 /**
823 * Determine whether or not interrupt events are pending on the
824 * the given interrupt source.
825 *
826 * \param isrc The interrupt source to query.
827 *
828 * \returns 0 if no events are pending, otherwise non-zero.
829 */
830 static int
xen_intr_source_pending(struct intsrc * isrc)831 xen_intr_source_pending(struct intsrc *isrc)
832 {
833 /*
834 * EventChannels are edge triggered and never masked.
835 * There can be no pending events.
836 */
837 return (0);
838 }
839
840 /**
841 * Perform configuration of an interrupt source.
842 *
843 * \param isrc The interrupt source to configure.
844 * \param trig Edge or level.
845 * \param pol Active high or low.
846 *
847 * \returns 0 if no events are pending, otherwise non-zero.
848 */
849 static int
xen_intr_config_intr(struct intsrc * isrc,enum intr_trigger trig,enum intr_polarity pol)850 xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
851 enum intr_polarity pol)
852 {
853 /* Configuration is only possible via the evtchn apis. */
854 return (ENODEV);
855 }
856
857 /**
858 * Configure CPU affinity for interrupt source event delivery.
859 *
860 * \param isrc The interrupt source to configure.
861 * \param apic_id The apic id of the CPU for handling future events.
862 *
863 * \returns 0 if successful, otherwise an errno.
864 */
865 static int
xen_intr_assign_cpu(struct intsrc * base_isrc,u_int apic_id)866 xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
867 {
868 #ifdef SMP
869 struct evtchn_bind_vcpu bind_vcpu;
870 struct xenisrc *isrc;
871 u_int to_cpu, vcpu_id;
872 int error, masked;
873
874 if (xen_vector_callback_enabled == 0)
875 return (EOPNOTSUPP);
876
877 to_cpu = apic_cpuid(apic_id);
878 vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id;
879 xen_intr_intrcnt_add(to_cpu);
880
881 mtx_lock(&xen_intr_isrc_lock);
882 isrc = (struct xenisrc *)base_isrc;
883 if (!is_valid_evtchn(isrc->xi_port)) {
884 mtx_unlock(&xen_intr_isrc_lock);
885 return (EINVAL);
886 }
887
888 /*
889 * Mask the event channel while binding it to prevent interrupt
890 * delivery with an inconsistent state in isrc->xi_cpu.
891 */
892 masked = evtchn_test_and_set_mask(isrc->xi_port);
893 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
894 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
895 /*
896 * Virtual IRQs are associated with a cpu by
897 * the Hypervisor at evtchn_bind_virq time, so
898 * all we need to do is update the per-CPU masks.
899 */
900 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
901 isrc->xi_cpu = to_cpu;
902 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
903 goto out;
904 }
905
906 bind_vcpu.port = isrc->xi_port;
907 bind_vcpu.vcpu = vcpu_id;
908
909 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
910 if (isrc->xi_cpu != to_cpu) {
911 if (error == 0) {
912 /* Commit to new binding by removing the old one. */
913 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
914 isrc->xi_cpu = to_cpu;
915 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
916 }
917 }
918
919 out:
920 if (masked == 0)
921 evtchn_unmask_port(isrc->xi_port);
922 mtx_unlock(&xen_intr_isrc_lock);
923 return (0);
924 #else
925 return (EOPNOTSUPP);
926 #endif
927 }
928
929 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
930 /*
931 * Mask a level triggered interrupt source.
932 *
933 * \param isrc The interrupt source to mask (if necessary).
934 * \param eoi If non-zero, perform any necessary end-of-interrupt
935 * acknowledgements.
936 */
937 static void
xen_intr_disable_source(struct intsrc * base_isrc,int eoi)938 xen_intr_disable_source(struct intsrc *base_isrc, int eoi)
939 {
940 struct xenisrc *isrc;
941
942 isrc = (struct xenisrc *)base_isrc;
943
944 /*
945 * NB: checking if the event channel is already masked is
946 * needed because the event channel user-space device
947 * masks event channels on it's filter as part of it's
948 * normal operation, and those shouldn't be automatically
949 * unmasked by the generic interrupt code. The event channel
950 * device will unmask them when needed.
951 */
952 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
953 }
954
955 /*
956 * Unmask a level triggered interrupt source.
957 *
958 * \param isrc The interrupt source to unmask (if necessary).
959 */
960 static void
xen_intr_enable_source(struct intsrc * base_isrc)961 xen_intr_enable_source(struct intsrc *base_isrc)
962 {
963 struct xenisrc *isrc;
964
965 isrc = (struct xenisrc *)base_isrc;
966
967 if (isrc->xi_masked == 0)
968 evtchn_unmask_port(isrc->xi_port);
969 }
970
971 /*
972 * Perform any necessary end-of-interrupt acknowledgements.
973 *
974 * \param isrc The interrupt source to EOI.
975 */
976 static void
xen_intr_eoi_source(struct intsrc * base_isrc)977 xen_intr_eoi_source(struct intsrc *base_isrc)
978 {
979 }
980
981 /*
982 * Enable and unmask the interrupt source.
983 *
984 * \param isrc The interrupt source to enable.
985 */
986 static void
xen_intr_enable_intr(struct intsrc * base_isrc)987 xen_intr_enable_intr(struct intsrc *base_isrc)
988 {
989 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
990
991 evtchn_unmask_port(isrc->xi_port);
992 }
993
994 /*------------------ Physical Interrupt Source PIC Functions -----------------*/
995 /*
996 * Mask a level triggered interrupt source.
997 *
998 * \param isrc The interrupt source to mask (if necessary).
999 * \param eoi If non-zero, perform any necessary end-of-interrupt
1000 * acknowledgements.
1001 */
1002 static void
xen_intr_pirq_disable_source(struct intsrc * base_isrc,int eoi)1003 xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi)
1004 {
1005 struct xenisrc *isrc;
1006
1007 isrc = (struct xenisrc *)base_isrc;
1008
1009 if (isrc->xi_edgetrigger == 0)
1010 evtchn_mask_port(isrc->xi_port);
1011 if (eoi == PIC_EOI)
1012 xen_intr_pirq_eoi_source(base_isrc);
1013 }
1014
1015 /*
1016 * Unmask a level triggered interrupt source.
1017 *
1018 * \param isrc The interrupt source to unmask (if necessary).
1019 */
1020 static void
xen_intr_pirq_enable_source(struct intsrc * base_isrc)1021 xen_intr_pirq_enable_source(struct intsrc *base_isrc)
1022 {
1023 struct xenisrc *isrc;
1024
1025 isrc = (struct xenisrc *)base_isrc;
1026
1027 if (isrc->xi_edgetrigger == 0)
1028 evtchn_unmask_port(isrc->xi_port);
1029 }
1030
1031 /*
1032 * Perform any necessary end-of-interrupt acknowledgements.
1033 *
1034 * \param isrc The interrupt source to EOI.
1035 */
1036 static void
xen_intr_pirq_eoi_source(struct intsrc * base_isrc)1037 xen_intr_pirq_eoi_source(struct intsrc *base_isrc)
1038 {
1039 struct xenisrc *isrc;
1040 int error;
1041
1042 isrc = (struct xenisrc *)base_isrc;
1043
1044 if (xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map)) {
1045 struct physdev_eoi eoi = { .irq = isrc->xi_pirq };
1046
1047 error = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
1048 if (error != 0)
1049 panic("Unable to EOI PIRQ#%d: %d\n",
1050 isrc->xi_pirq, error);
1051 }
1052 }
1053
1054 /*
1055 * Enable and unmask the interrupt source.
1056 *
1057 * \param isrc The interrupt source to enable.
1058 */
1059 static void
xen_intr_pirq_enable_intr(struct intsrc * base_isrc)1060 xen_intr_pirq_enable_intr(struct intsrc *base_isrc)
1061 {
1062 struct xenisrc *isrc;
1063 struct evtchn_bind_pirq bind_pirq;
1064 struct physdev_irq_status_query irq_status;
1065 int error;
1066
1067 isrc = (struct xenisrc *)base_isrc;
1068
1069 if (!xen_intr_pirq_eoi_map_enabled) {
1070 irq_status.irq = isrc->xi_pirq;
1071 error = HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query,
1072 &irq_status);
1073 if (error)
1074 panic("unable to get status of IRQ#%d", isrc->xi_pirq);
1075
1076 if (irq_status.flags & XENIRQSTAT_needs_eoi) {
1077 /*
1078 * Since the dynamic PIRQ EOI map is not available
1079 * mark the PIRQ as needing EOI unconditionally.
1080 */
1081 xen_set_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map);
1082 }
1083 }
1084
1085 bind_pirq.pirq = isrc->xi_pirq;
1086 bind_pirq.flags = isrc->xi_edgetrigger ? 0 : BIND_PIRQ__WILL_SHARE;
1087 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
1088 if (error)
1089 panic("unable to bind IRQ#%d", isrc->xi_pirq);
1090
1091 isrc->xi_port = bind_pirq.port;
1092
1093 mtx_lock(&xen_intr_isrc_lock);
1094 KASSERT((xen_intr_port_to_isrc[bind_pirq.port] == NULL),
1095 ("trying to override an already setup event channel port"));
1096 xen_intr_port_to_isrc[bind_pirq.port] = isrc;
1097 mtx_unlock(&xen_intr_isrc_lock);
1098
1099 evtchn_unmask_port(isrc->xi_port);
1100 }
1101
1102 /*
1103 * Disable an interrupt source.
1104 *
1105 * \param isrc The interrupt source to disable.
1106 */
1107 static void
xen_intr_pirq_disable_intr(struct intsrc * base_isrc)1108 xen_intr_pirq_disable_intr(struct intsrc *base_isrc)
1109 {
1110 struct xenisrc *isrc;
1111 struct evtchn_close close;
1112 int error;
1113
1114 isrc = (struct xenisrc *)base_isrc;
1115
1116 evtchn_mask_port(isrc->xi_port);
1117
1118 close.port = isrc->xi_port;
1119 error = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
1120 if (error)
1121 panic("unable to close event channel %d IRQ#%d",
1122 isrc->xi_port, isrc->xi_pirq);
1123
1124 mtx_lock(&xen_intr_isrc_lock);
1125 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
1126 mtx_unlock(&xen_intr_isrc_lock);
1127
1128 isrc->xi_port = 0;
1129 }
1130
1131 /**
1132 * Perform configuration of an interrupt source.
1133 *
1134 * \param isrc The interrupt source to configure.
1135 * \param trig Edge or level.
1136 * \param pol Active high or low.
1137 *
1138 * \returns 0 if no events are pending, otherwise non-zero.
1139 */
1140 static int
xen_intr_pirq_config_intr(struct intsrc * base_isrc,enum intr_trigger trig,enum intr_polarity pol)1141 xen_intr_pirq_config_intr(struct intsrc *base_isrc, enum intr_trigger trig,
1142 enum intr_polarity pol)
1143 {
1144 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1145 struct physdev_setup_gsi setup_gsi;
1146 int error;
1147
1148 KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM),
1149 ("%s: Conforming trigger or polarity\n", __func__));
1150
1151 setup_gsi.gsi = isrc->xi_pirq;
1152 setup_gsi.triggering = trig == INTR_TRIGGER_EDGE ? 0 : 1;
1153 setup_gsi.polarity = pol == INTR_POLARITY_HIGH ? 0 : 1;
1154
1155 error = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi);
1156 if (error == -XEN_EEXIST) {
1157 if ((isrc->xi_edgetrigger && (trig != INTR_TRIGGER_EDGE)) ||
1158 (isrc->xi_activehi && (pol != INTR_POLARITY_HIGH)))
1159 panic("unable to reconfigure interrupt IRQ#%d",
1160 isrc->xi_pirq);
1161 error = 0;
1162 }
1163 if (error)
1164 panic("unable to configure IRQ#%d\n", isrc->xi_pirq);
1165
1166 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1167 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1168
1169 return (0);
1170 }
1171
1172 /*--------------------------- Public Functions -------------------------------*/
1173 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
1174 int
xen_intr_bind_local_port(device_t dev,evtchn_port_t local_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1175 xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
1176 driver_filter_t filter, driver_intr_t handler, void *arg,
1177 enum intr_type flags, xen_intr_handle_t *port_handlep)
1178 {
1179 struct xenisrc *isrc;
1180 int error;
1181
1182 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev,
1183 filter, handler, arg, flags, port_handlep);
1184 if (error != 0)
1185 return (error);
1186
1187 /*
1188 * The Event Channel API didn't open this port, so it is not
1189 * responsible for closing it automatically on unbind.
1190 */
1191 isrc->xi_close = 0;
1192 return (0);
1193 }
1194
1195 int
xen_intr_alloc_and_bind_local_port(device_t dev,u_int remote_domain,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1196 xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
1197 driver_filter_t filter, driver_intr_t handler, void *arg,
1198 enum intr_type flags, xen_intr_handle_t *port_handlep)
1199 {
1200 struct xenisrc *isrc;
1201 struct evtchn_alloc_unbound alloc_unbound;
1202 int error;
1203
1204 alloc_unbound.dom = DOMID_SELF;
1205 alloc_unbound.remote_dom = remote_domain;
1206 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
1207 &alloc_unbound);
1208 if (error != 0) {
1209 /*
1210 * XXX Trap Hypercall error code Linuxisms in
1211 * the HYPERCALL layer.
1212 */
1213 return (-error);
1214 }
1215
1216 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1217 dev, filter, handler, arg, flags,
1218 port_handlep);
1219 if (error != 0) {
1220 evtchn_close_t close = { .port = alloc_unbound.port };
1221 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1222 panic("EVTCHNOP_close failed");
1223 return (error);
1224 }
1225
1226 isrc->xi_close = 1;
1227 return (0);
1228 }
1229
1230 int
xen_intr_bind_remote_port(device_t dev,u_int remote_domain,u_int remote_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1231 xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1232 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1233 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1234 {
1235 struct xenisrc *isrc;
1236 struct evtchn_bind_interdomain bind_interdomain;
1237 int error;
1238
1239 bind_interdomain.remote_dom = remote_domain;
1240 bind_interdomain.remote_port = remote_port;
1241 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1242 &bind_interdomain);
1243 if (error != 0) {
1244 /*
1245 * XXX Trap Hypercall error code Linuxisms in
1246 * the HYPERCALL layer.
1247 */
1248 return (-error);
1249 }
1250
1251 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1252 EVTCHN_TYPE_PORT, dev, filter, handler,
1253 arg, flags, port_handlep);
1254 if (error) {
1255 evtchn_close_t close = { .port = bind_interdomain.local_port };
1256 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1257 panic("EVTCHNOP_close failed");
1258 return (error);
1259 }
1260
1261 /*
1262 * The Event Channel API opened this port, so it is
1263 * responsible for closing it automatically on unbind.
1264 */
1265 isrc->xi_close = 1;
1266 return (0);
1267 }
1268
1269 int
xen_intr_bind_virq(device_t dev,u_int virq,u_int cpu,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1270 xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1271 driver_filter_t filter, driver_intr_t handler, void *arg,
1272 enum intr_type flags, xen_intr_handle_t *port_handlep)
1273 {
1274 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1275 struct xenisrc *isrc;
1276 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
1277 int error;
1278
1279 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1280 xen_intr_intrcnt_add(cpu);
1281
1282 isrc = NULL;
1283 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1284 if (error != 0) {
1285 /*
1286 * XXX Trap Hypercall error code Linuxisms in
1287 * the HYPERCALL layer.
1288 */
1289 return (-error);
1290 }
1291
1292 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev,
1293 filter, handler, arg, flags, port_handlep);
1294
1295 #ifdef SMP
1296 if (error == 0)
1297 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1298 #endif
1299
1300 if (error != 0) {
1301 evtchn_close_t close = { .port = bind_virq.port };
1302
1303 xen_intr_unbind(*port_handlep);
1304 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1305 panic("EVTCHNOP_close failed");
1306 return (error);
1307 }
1308
1309 #ifdef SMP
1310 if (isrc->xi_cpu != cpu) {
1311 /*
1312 * Too early in the boot process for the generic interrupt
1313 * code to perform the binding. Update our event channel
1314 * masks manually so events can't fire on the wrong cpu
1315 * during AP startup.
1316 */
1317 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1318 }
1319 #endif
1320
1321 /*
1322 * The Event Channel API opened this port, so it is
1323 * responsible for closing it automatically on unbind.
1324 */
1325 isrc->xi_close = 1;
1326 isrc->xi_virq = virq;
1327
1328 return (0);
1329 }
1330
1331 int
xen_intr_alloc_and_bind_ipi(device_t dev,u_int cpu,driver_filter_t filter,enum intr_type flags,xen_intr_handle_t * port_handlep)1332 xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu,
1333 driver_filter_t filter, enum intr_type flags,
1334 xen_intr_handle_t *port_handlep)
1335 {
1336 #ifdef SMP
1337 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1338 struct xenisrc *isrc;
1339 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
1340 int error;
1341
1342 /* Ensure the target CPU is ready to handle evtchn interrupts. */
1343 xen_intr_intrcnt_add(cpu);
1344
1345 isrc = NULL;
1346 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1347 if (error != 0) {
1348 /*
1349 * XXX Trap Hypercall error code Linuxisms in
1350 * the HYPERCALL layer.
1351 */
1352 return (-error);
1353 }
1354
1355 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1356 dev, filter, NULL, NULL, flags,
1357 port_handlep);
1358 if (error == 0)
1359 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1360
1361 if (error != 0) {
1362 evtchn_close_t close = { .port = bind_ipi.port };
1363
1364 xen_intr_unbind(*port_handlep);
1365 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1366 panic("EVTCHNOP_close failed");
1367 return (error);
1368 }
1369
1370 if (isrc->xi_cpu != cpu) {
1371 /*
1372 * Too early in the boot process for the generic interrupt
1373 * code to perform the binding. Update our event channel
1374 * masks manually so events can't fire on the wrong cpu
1375 * during AP startup.
1376 */
1377 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1378 }
1379
1380 /*
1381 * The Event Channel API opened this port, so it is
1382 * responsible for closing it automatically on unbind.
1383 */
1384 isrc->xi_close = 1;
1385 return (0);
1386 #else
1387 return (EOPNOTSUPP);
1388 #endif
1389 }
1390
1391 int
xen_register_pirq(int vector,enum intr_trigger trig,enum intr_polarity pol)1392 xen_register_pirq(int vector, enum intr_trigger trig, enum intr_polarity pol)
1393 {
1394 struct physdev_map_pirq map_pirq;
1395 struct xenisrc *isrc;
1396 int error;
1397
1398 if (vector == 0)
1399 return (EINVAL);
1400
1401 if (bootverbose)
1402 printf("xen: register IRQ#%d\n", vector);
1403
1404 map_pirq.domid = DOMID_SELF;
1405 map_pirq.type = MAP_PIRQ_TYPE_GSI;
1406 map_pirq.index = vector;
1407 map_pirq.pirq = vector;
1408
1409 error = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_pirq);
1410 if (error) {
1411 printf("xen: unable to map IRQ#%d\n", vector);
1412 return (error);
1413 }
1414
1415 mtx_lock(&xen_intr_isrc_lock);
1416 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector);
1417 mtx_unlock(&xen_intr_isrc_lock);
1418 KASSERT((isrc != NULL), ("xen: unable to allocate isrc for interrupt"));
1419 isrc->xi_pirq = vector;
1420 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1421 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1422
1423 return (0);
1424 }
1425
1426 int
xen_register_msi(device_t dev,int vector,int count)1427 xen_register_msi(device_t dev, int vector, int count)
1428 {
1429 struct physdev_map_pirq msi_irq;
1430 struct xenisrc *isrc;
1431 int ret;
1432
1433 memset(&msi_irq, 0, sizeof(msi_irq));
1434 msi_irq.domid = DOMID_SELF;
1435 msi_irq.type = count == 1 ?
1436 MAP_PIRQ_TYPE_MSI_SEG : MAP_PIRQ_TYPE_MULTI_MSI;
1437 msi_irq.index = -1;
1438 msi_irq.pirq = -1;
1439 msi_irq.bus = pci_get_bus(dev) | (pci_get_domain(dev) << 16);
1440 msi_irq.devfn = (pci_get_slot(dev) << 3) | pci_get_function(dev);
1441 msi_irq.entry_nr = count;
1442
1443 ret = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &msi_irq);
1444 if (ret != 0)
1445 return (ret);
1446 if (count != msi_irq.entry_nr) {
1447 panic("unable to setup all requested MSI vectors "
1448 "(expected %d got %d)", count, msi_irq.entry_nr);
1449 }
1450
1451 mtx_lock(&xen_intr_isrc_lock);
1452 for (int i = 0; i < count; i++) {
1453 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector + i);
1454 KASSERT(isrc != NULL,
1455 ("xen: unable to allocate isrc for interrupt"));
1456 isrc->xi_pirq = msi_irq.pirq + i;
1457 /* MSI interrupts are always edge triggered */
1458 isrc->xi_edgetrigger = 1;
1459 }
1460 mtx_unlock(&xen_intr_isrc_lock);
1461
1462 return (0);
1463 }
1464
1465 int
xen_release_msi(int vector)1466 xen_release_msi(int vector)
1467 {
1468 struct physdev_unmap_pirq unmap;
1469 struct xenisrc *isrc;
1470 int ret;
1471
1472 isrc = (struct xenisrc *)intr_lookup_source(vector);
1473 if (isrc == NULL)
1474 return (ENXIO);
1475
1476 unmap.pirq = isrc->xi_pirq;
1477 ret = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap);
1478 if (ret != 0)
1479 return (ret);
1480
1481 xen_intr_release_isrc(isrc);
1482
1483 return (0);
1484 }
1485
1486 int
xen_intr_describe(xen_intr_handle_t port_handle,const char * fmt,...)1487 xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1488 {
1489 char descr[MAXCOMLEN + 1];
1490 struct xenisrc *isrc;
1491 va_list ap;
1492
1493 isrc = xen_intr_isrc(port_handle);
1494 if (isrc == NULL)
1495 return (EINVAL);
1496
1497 va_start(ap, fmt);
1498 vsnprintf(descr, sizeof(descr), fmt, ap);
1499 va_end(ap);
1500 return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr));
1501 }
1502
1503 void
xen_intr_unbind(xen_intr_handle_t * port_handlep)1504 xen_intr_unbind(xen_intr_handle_t *port_handlep)
1505 {
1506 struct xenisrc *isrc;
1507
1508 KASSERT(port_handlep != NULL,
1509 ("NULL xen_intr_handle_t passed to xen_intr_unbind"));
1510
1511 isrc = xen_intr_isrc(*port_handlep);
1512 *port_handlep = NULL;
1513 if (isrc == NULL)
1514 return;
1515
1516 if (isrc->xi_cookie != NULL)
1517 intr_remove_handler(isrc->xi_cookie);
1518 xen_intr_release_isrc(isrc);
1519 }
1520
1521 void
xen_intr_signal(xen_intr_handle_t handle)1522 xen_intr_signal(xen_intr_handle_t handle)
1523 {
1524 struct xenisrc *isrc;
1525
1526 isrc = xen_intr_isrc(handle);
1527 if (isrc != NULL) {
1528 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1529 isrc->xi_type == EVTCHN_TYPE_IPI,
1530 ("evtchn_signal on something other than a local port"));
1531 struct evtchn_send send = { .port = isrc->xi_port };
1532 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1533 }
1534 }
1535
1536 evtchn_port_t
xen_intr_port(xen_intr_handle_t handle)1537 xen_intr_port(xen_intr_handle_t handle)
1538 {
1539 struct xenisrc *isrc;
1540
1541 isrc = xen_intr_isrc(handle);
1542 if (isrc == NULL)
1543 return (0);
1544
1545 return (isrc->xi_port);
1546 }
1547
1548 int
xen_intr_add_handler(device_t dev,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t handle)1549 xen_intr_add_handler(device_t dev, driver_filter_t filter,
1550 driver_intr_t handler, void *arg, enum intr_type flags,
1551 xen_intr_handle_t handle)
1552 {
1553 struct xenisrc *isrc;
1554 int error;
1555
1556 isrc = xen_intr_isrc(handle);
1557 if (isrc == NULL || isrc->xi_cookie != NULL)
1558 return (EINVAL);
1559
1560 error = intr_add_handler(device_get_nameunit(dev), isrc->xi_vector,
1561 filter, handler, arg, flags|INTR_EXCL, &isrc->xi_cookie);
1562 if (error != 0) {
1563 device_printf(dev,
1564 "xen_intr_add_handler: intr_add_handler failed: %d\n",
1565 error);
1566 }
1567
1568 return (error);
1569 }
1570
1571 #ifdef DDB
1572 static const char *
xen_intr_print_type(enum evtchn_type type)1573 xen_intr_print_type(enum evtchn_type type)
1574 {
1575 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
1576 [EVTCHN_TYPE_UNBOUND] = "UNBOUND",
1577 [EVTCHN_TYPE_PIRQ] = "PIRQ",
1578 [EVTCHN_TYPE_VIRQ] = "VIRQ",
1579 [EVTCHN_TYPE_IPI] = "IPI",
1580 [EVTCHN_TYPE_PORT] = "PORT",
1581 };
1582
1583 if (type >= EVTCHN_TYPE_COUNT)
1584 return ("UNKNOWN");
1585
1586 return (evtchn_type_to_string[type]);
1587 }
1588
1589 static void
xen_intr_dump_port(struct xenisrc * isrc)1590 xen_intr_dump_port(struct xenisrc *isrc)
1591 {
1592 struct xen_intr_pcpu_data *pcpu;
1593 shared_info_t *s = HYPERVISOR_shared_info;
1594 int i;
1595
1596 db_printf("Port %d Type: %s\n",
1597 isrc->xi_port, xen_intr_print_type(isrc->xi_type));
1598 if (isrc->xi_type == EVTCHN_TYPE_PIRQ) {
1599 db_printf("\tPirq: %d ActiveHi: %d EdgeTrigger: %d "
1600 "NeedsEOI: %d\n",
1601 isrc->xi_pirq, isrc->xi_activehi, isrc->xi_edgetrigger,
1602 !!xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map));
1603 }
1604 if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
1605 db_printf("\tVirq: %d\n", isrc->xi_virq);
1606
1607 db_printf("\tMasked: %d Pending: %d\n",
1608 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
1609 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
1610
1611 db_printf("\tPer-CPU Masks: ");
1612 CPU_FOREACH(i) {
1613 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
1614 db_printf("cpu#%d: %d ", i,
1615 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
1616 }
1617 db_printf("\n");
1618 }
1619
DB_SHOW_COMMAND(xen_evtchn,db_show_xen_evtchn)1620 DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
1621 {
1622 int i;
1623
1624 if (!xen_domain()) {
1625 db_printf("Only available on Xen guests\n");
1626 return;
1627 }
1628
1629 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1630 struct xenisrc *isrc;
1631
1632 isrc = xen_intr_port_to_isrc[i];
1633 if (isrc == NULL)
1634 continue;
1635
1636 xen_intr_dump_port(isrc);
1637 }
1638 }
1639 #endif /* DDB */
1640