1 /******************************************************************************
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32 #include <sys/cdefs.h>
33 #include "opt_ddb.h"
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/bus.h>
38 #include <sys/malloc.h>
39 #include <sys/kernel.h>
40 #include <sys/limits.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/interrupt.h>
44 #include <sys/pcpu.h>
45 #include <sys/smp.h>
46 #include <sys/refcount.h>
47
48 #include <vm/vm.h>
49 #include <vm/pmap.h>
50
51 #include <machine/intr_machdep.h>
52 #include <x86/apicvar.h>
53 #include <x86/apicreg.h>
54 #include <machine/smp.h>
55 #include <machine/stdarg.h>
56
57 #include <machine/xen/synch_bitops.h>
58 #include <machine/xen/xen-os.h>
59
60 #include <xen/xen-os.h>
61 #include <xen/hvm.h>
62 #include <xen/hypervisor.h>
63 #include <xen/xen_intr.h>
64 #include <xen/evtchn/evtchnvar.h>
65
66 #include <dev/xen/xenpci/xenpcivar.h>
67 #include <dev/pci/pcivar.h>
68
69 #ifdef DDB
70 #include <ddb/ddb.h>
71 #endif
72
73 static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
74
75 static u_int first_evtchn_irq;
76
77 /**
78 * Per-cpu event channel processing state.
79 */
80 struct xen_intr_pcpu_data {
81 /**
82 * The last event channel bitmap section (level one bit) processed.
83 * This is used to ensure we scan all ports before
84 * servicing an already servied port again.
85 */
86 u_int last_processed_l1i;
87
88 /**
89 * The last event channel processed within the event channel
90 * bitmap being scanned.
91 */
92 u_int last_processed_l2i;
93
94 /** Pointer to this CPU's interrupt statistic counter. */
95 u_long *evtchn_intrcnt;
96
97 /**
98 * A bitmap of ports that can be serviced from this CPU.
99 * A set bit means interrupt handling is enabled.
100 */
101 u_long evtchn_enabled[sizeof(u_long) * 8];
102 };
103
104 /*
105 * Start the scan at port 0 by initializing the last scanned
106 * location as the highest numbered event channel port.
107 */
108 DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
109 .last_processed_l1i = LONG_BIT - 1,
110 .last_processed_l2i = LONG_BIT - 1
111 };
112
113 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
114
115 #define XEN_EEXIST 17 /* Xen "already exists" error */
116 #define XEN_ALLOCATE_VECTOR 0 /* Allocate a vector for this event channel */
117 #define XEN_INVALID_EVTCHN 0 /* Invalid event channel */
118
119 #define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN)
120
121 struct xenisrc {
122 struct intsrc xi_intsrc;
123 enum evtchn_type xi_type;
124 int xi_cpu; /* VCPU for delivery. */
125 int xi_vector; /* Global isrc vector number. */
126 evtchn_port_t xi_port;
127 int xi_pirq;
128 int xi_virq;
129 void *xi_cookie;
130 u_int xi_close:1; /* close on unbind? */
131 u_int xi_activehi:1;
132 u_int xi_edgetrigger:1;
133 u_int xi_masked:1;
134 volatile u_int xi_refcount;
135 };
136
137 static void xen_intr_suspend(struct pic *);
138 static void xen_intr_resume(struct pic *, bool suspend_cancelled);
139 static void xen_intr_enable_source(struct intsrc *isrc);
140 static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
141 static void xen_intr_eoi_source(struct intsrc *isrc);
142 static void xen_intr_enable_intr(struct intsrc *isrc);
143 static void xen_intr_disable_intr(struct intsrc *isrc);
144 static int xen_intr_vector(struct intsrc *isrc);
145 static int xen_intr_source_pending(struct intsrc *isrc);
146 static int xen_intr_config_intr(struct intsrc *isrc,
147 enum intr_trigger trig, enum intr_polarity pol);
148 static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
149
150 static void xen_intr_pirq_enable_source(struct intsrc *isrc);
151 static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi);
152 static void xen_intr_pirq_eoi_source(struct intsrc *isrc);
153 static void xen_intr_pirq_enable_intr(struct intsrc *isrc);
154 static void xen_intr_pirq_disable_intr(struct intsrc *isrc);
155 static int xen_intr_pirq_config_intr(struct intsrc *isrc,
156 enum intr_trigger trig, enum intr_polarity pol);
157
158 /**
159 * PIC interface for all event channel port types except physical IRQs.
160 */
161 struct pic xen_intr_pic = {
162 .pic_enable_source = xen_intr_enable_source,
163 .pic_disable_source = xen_intr_disable_source,
164 .pic_eoi_source = xen_intr_eoi_source,
165 .pic_enable_intr = xen_intr_enable_intr,
166 .pic_disable_intr = xen_intr_disable_intr,
167 .pic_vector = xen_intr_vector,
168 .pic_source_pending = xen_intr_source_pending,
169 .pic_suspend = xen_intr_suspend,
170 .pic_resume = xen_intr_resume,
171 .pic_config_intr = xen_intr_config_intr,
172 .pic_assign_cpu = xen_intr_assign_cpu
173 };
174
175 /**
176 * PIC interface for all event channel representing
177 * physical interrupt sources.
178 */
179 struct pic xen_intr_pirq_pic = {
180 #ifdef __amd64__
181 .pic_register_sources = xenpv_register_pirqs,
182 #endif
183 .pic_enable_source = xen_intr_pirq_enable_source,
184 .pic_disable_source = xen_intr_pirq_disable_source,
185 .pic_eoi_source = xen_intr_pirq_eoi_source,
186 .pic_enable_intr = xen_intr_pirq_enable_intr,
187 .pic_disable_intr = xen_intr_pirq_disable_intr,
188 .pic_vector = xen_intr_vector,
189 .pic_source_pending = xen_intr_source_pending,
190 .pic_config_intr = xen_intr_pirq_config_intr,
191 .pic_assign_cpu = xen_intr_assign_cpu
192 };
193
194 static struct mtx xen_intr_isrc_lock;
195 static u_int xen_intr_auto_vector_count;
196 static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
197 static u_long *xen_intr_pirq_eoi_map;
198 static boolean_t xen_intr_pirq_eoi_map_enabled;
199
200 /*------------------------- Private Functions --------------------------------*/
201 /**
202 * Disable signal delivery for an event channel port on the
203 * specified CPU.
204 *
205 * \param port The event channel port to mask.
206 *
207 * This API is used to manage the port<=>CPU binding of event
208 * channel handlers.
209 *
210 * \note This operation does not preclude reception of an event
211 * for this event channel on another CPU. To mask the
212 * event channel globally, use evtchn_mask().
213 */
214 static inline void
evtchn_cpu_mask_port(u_int cpu,evtchn_port_t port)215 evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
216 {
217 struct xen_intr_pcpu_data *pcpu;
218
219 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
220 xen_clear_bit(port, pcpu->evtchn_enabled);
221 }
222
223 /**
224 * Enable signal delivery for an event channel port on the
225 * specified CPU.
226 *
227 * \param port The event channel port to unmask.
228 *
229 * This API is used to manage the port<=>CPU binding of event
230 * channel handlers.
231 *
232 * \note This operation does not guarantee that event delivery
233 * is enabled for this event channel port. The port must
234 * also be globally enabled. See evtchn_unmask().
235 */
236 static inline void
evtchn_cpu_unmask_port(u_int cpu,evtchn_port_t port)237 evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
238 {
239 struct xen_intr_pcpu_data *pcpu;
240
241 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
242 xen_set_bit(port, pcpu->evtchn_enabled);
243 }
244
245 /**
246 * Allocate and register a per-cpu Xen upcall interrupt counter.
247 *
248 * \param cpu The cpu for which to register this interrupt count.
249 */
250 static void
xen_intr_intrcnt_add(u_int cpu)251 xen_intr_intrcnt_add(u_int cpu)
252 {
253 char buf[MAXCOMLEN + 1];
254 struct xen_intr_pcpu_data *pcpu;
255
256 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
257 if (pcpu->evtchn_intrcnt != NULL)
258 return;
259
260 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
261 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
262 }
263
264 /**
265 * Search for an already allocated but currently unused Xen interrupt
266 * source object.
267 *
268 * \param type Restrict the search to interrupt sources of the given
269 * type.
270 *
271 * \return A pointer to a free Xen interrupt source object or NULL.
272 */
273 static struct xenisrc *
xen_intr_find_unused_isrc(enum evtchn_type type)274 xen_intr_find_unused_isrc(enum evtchn_type type)
275 {
276 int isrc_idx;
277
278 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
279
280 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) {
281 struct xenisrc *isrc;
282 u_int vector;
283
284 vector = first_evtchn_irq + isrc_idx;
285 isrc = (struct xenisrc *)intr_lookup_source(vector);
286 if (isrc != NULL
287 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
288 KASSERT(isrc->xi_intsrc.is_handlers == 0,
289 ("Free evtchn still has handlers"));
290 isrc->xi_type = type;
291 return (isrc);
292 }
293 }
294 return (NULL);
295 }
296
297 /**
298 * Allocate a Xen interrupt source object.
299 *
300 * \param type The type of interrupt source to create.
301 *
302 * \return A pointer to a newly allocated Xen interrupt source
303 * object or NULL.
304 */
305 static struct xenisrc *
xen_intr_alloc_isrc(enum evtchn_type type,int vector)306 xen_intr_alloc_isrc(enum evtchn_type type, int vector)
307 {
308 static int warned;
309 struct xenisrc *isrc;
310
311 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
312
313 if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) {
314 if (!warned) {
315 warned = 1;
316 printf("xen_intr_alloc: Event channels exhausted.\n");
317 }
318 return (NULL);
319 }
320
321 if (type != EVTCHN_TYPE_PIRQ) {
322 vector = first_evtchn_irq + xen_intr_auto_vector_count;
323 xen_intr_auto_vector_count++;
324 }
325
326 KASSERT((intr_lookup_source(vector) == NULL),
327 ("Trying to use an already allocated vector"));
328
329 mtx_unlock(&xen_intr_isrc_lock);
330 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
331 isrc->xi_intsrc.is_pic =
332 (type == EVTCHN_TYPE_PIRQ) ? &xen_intr_pirq_pic : &xen_intr_pic;
333 isrc->xi_vector = vector;
334 isrc->xi_type = type;
335 intr_register_source(&isrc->xi_intsrc);
336 mtx_lock(&xen_intr_isrc_lock);
337
338 return (isrc);
339 }
340
341 /**
342 * Attempt to free an active Xen interrupt source object.
343 *
344 * \param isrc The interrupt source object to release.
345 *
346 * \returns EBUSY if the source is still in use, otherwise 0.
347 */
348 static int
xen_intr_release_isrc(struct xenisrc * isrc)349 xen_intr_release_isrc(struct xenisrc *isrc)
350 {
351
352 mtx_lock(&xen_intr_isrc_lock);
353 KASSERT(isrc->xi_intsrc.is_handlers == 0,
354 ("Release called, but xenisrc still in use"));
355 evtchn_mask_port(isrc->xi_port);
356 evtchn_clear_port(isrc->xi_port);
357
358 /* Rebind port to CPU 0. */
359 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
360 evtchn_cpu_unmask_port(0, isrc->xi_port);
361
362 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
363 struct evtchn_close close = { .port = isrc->xi_port };
364 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
365 panic("EVTCHNOP_close failed");
366 }
367
368 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
369 isrc->xi_cpu = 0;
370 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
371 isrc->xi_port = 0;
372 isrc->xi_cookie = NULL;
373 mtx_unlock(&xen_intr_isrc_lock);
374 return (0);
375 }
376
377 /**
378 * Associate an interrupt handler with an already allocated local Xen
379 * event channel port.
380 *
381 * \param isrcp The returned Xen interrupt object associated with
382 * the specified local port.
383 * \param local_port The event channel to bind.
384 * \param type The event channel type of local_port.
385 * \param intr_owner The device making this bind request.
386 * \param filter An interrupt filter handler. Specify NULL
387 * to always dispatch to the ithread handler.
388 * \param handler An interrupt ithread handler. Optional (can
389 * specify NULL) if all necessary event actions
390 * are performed by filter.
391 * \param arg Argument to present to both filter and handler.
392 * \param irqflags Interrupt handler flags. See sys/bus.h.
393 * \param handlep Pointer to an opaque handle used to manage this
394 * registration.
395 *
396 * \returns 0 on success, otherwise an errno.
397 */
398 static int
xen_intr_bind_isrc(struct xenisrc ** isrcp,evtchn_port_t local_port,enum evtchn_type type,const char * intr_owner,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)399 xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
400 enum evtchn_type type, const char *intr_owner, driver_filter_t filter,
401 driver_intr_t handler, void *arg, enum intr_type flags,
402 xen_intr_handle_t *port_handlep)
403 {
404 struct xenisrc *isrc;
405 int error;
406
407 *isrcp = NULL;
408 if (port_handlep == NULL) {
409 printf("%s: xen_intr_bind_isrc: Bad event handle\n",
410 intr_owner);
411 return (EINVAL);
412 }
413
414 mtx_lock(&xen_intr_isrc_lock);
415 isrc = xen_intr_find_unused_isrc(type);
416 if (isrc == NULL) {
417 isrc = xen_intr_alloc_isrc(type, XEN_ALLOCATE_VECTOR);
418 if (isrc == NULL) {
419 mtx_unlock(&xen_intr_isrc_lock);
420 return (ENOSPC);
421 }
422 }
423 isrc->xi_port = local_port;
424 xen_intr_port_to_isrc[local_port] = isrc;
425 refcount_init(&isrc->xi_refcount, 1);
426 mtx_unlock(&xen_intr_isrc_lock);
427
428 /* Assign the opaque handler (the event channel port) */
429 *port_handlep = &isrc->xi_vector;
430
431 #ifdef SMP
432 if (type == EVTCHN_TYPE_PORT) {
433 /*
434 * By default all interrupts are assigned to vCPU#0
435 * unless specified otherwise, so shuffle them to balance
436 * the interrupt load.
437 */
438 xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu(0));
439 }
440 #endif
441
442 if (filter == NULL && handler == NULL) {
443 /*
444 * No filter/handler provided, leave the event channel
445 * masked and without a valid handler, the caller is
446 * in charge of setting that up.
447 */
448 *isrcp = isrc;
449 return (0);
450 }
451
452 error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags,
453 *port_handlep);
454 if (error != 0) {
455 xen_intr_release_isrc(isrc);
456 return (error);
457 }
458 *isrcp = isrc;
459 return (0);
460 }
461
462 /**
463 * Lookup a Xen interrupt source object given an interrupt binding handle.
464 *
465 * \param handle A handle initialized by a previous call to
466 * xen_intr_bind_isrc().
467 *
468 * \returns A pointer to the Xen interrupt source object associated
469 * with the given interrupt handle. NULL if no association
470 * currently exists.
471 */
472 static struct xenisrc *
xen_intr_isrc(xen_intr_handle_t handle)473 xen_intr_isrc(xen_intr_handle_t handle)
474 {
475 int vector;
476
477 if (handle == NULL)
478 return (NULL);
479
480 vector = *(int *)handle;
481 KASSERT(vector >= first_evtchn_irq &&
482 vector < (first_evtchn_irq + xen_intr_auto_vector_count),
483 ("Xen interrupt vector is out of range"));
484
485 return ((struct xenisrc *)intr_lookup_source(vector));
486 }
487
488 /**
489 * Determine the event channel ports at the given section of the
490 * event port bitmap which have pending events for the given cpu.
491 *
492 * \param pcpu The Xen interrupt pcpu data for the cpu being queried.
493 * \param sh The Xen shared info area.
494 * \param idx The index of the section of the event channel bitmap to
495 * inspect.
496 *
497 * \returns A u_long with bits set for every event channel with pending
498 * events.
499 */
500 static inline u_long
xen_intr_active_ports(struct xen_intr_pcpu_data * pcpu,shared_info_t * sh,u_int idx)501 xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
502 u_int idx)
503 {
504
505 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
506 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
507 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
508 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
509 return (sh->evtchn_pending[idx]
510 & ~sh->evtchn_mask[idx]
511 & pcpu->evtchn_enabled[idx]);
512 }
513
514 /**
515 * Interrupt handler for processing all Xen event channel events.
516 *
517 * \param trap_frame The trap frame context for the current interrupt.
518 */
519 void
xen_intr_handle_upcall(struct trapframe * trap_frame)520 xen_intr_handle_upcall(struct trapframe *trap_frame)
521 {
522 u_int l1i, l2i, port, cpu;
523 u_long masked_l1, masked_l2;
524 struct xenisrc *isrc;
525 shared_info_t *s;
526 vcpu_info_t *v;
527 struct xen_intr_pcpu_data *pc;
528 u_long l1, l2;
529
530 /*
531 * Disable preemption in order to always check and fire events
532 * on the right vCPU
533 */
534 critical_enter();
535
536 cpu = PCPU_GET(cpuid);
537 pc = DPCPU_PTR(xen_intr_pcpu);
538 s = HYPERVISOR_shared_info;
539 v = DPCPU_GET(vcpu_info);
540
541 if (xen_hvm_domain() && !xen_vector_callback_enabled) {
542 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
543 }
544
545 v->evtchn_upcall_pending = 0;
546
547 #if 0
548 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
549 /* Clear master flag /before/ clearing selector flag. */
550 wmb();
551 #endif
552 #endif
553
554 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
555
556 l1i = pc->last_processed_l1i;
557 l2i = pc->last_processed_l2i;
558 (*pc->evtchn_intrcnt)++;
559
560 while (l1 != 0) {
561 l1i = (l1i + 1) % LONG_BIT;
562 masked_l1 = l1 & ((~0UL) << l1i);
563
564 if (masked_l1 == 0) {
565 /*
566 * if we masked out all events, wrap around
567 * to the beginning.
568 */
569 l1i = LONG_BIT - 1;
570 l2i = LONG_BIT - 1;
571 continue;
572 }
573 l1i = ffsl(masked_l1) - 1;
574
575 do {
576 l2 = xen_intr_active_ports(pc, s, l1i);
577
578 l2i = (l2i + 1) % LONG_BIT;
579 masked_l2 = l2 & ((~0UL) << l2i);
580
581 if (masked_l2 == 0) {
582 /* if we masked out all events, move on */
583 l2i = LONG_BIT - 1;
584 break;
585 }
586 l2i = ffsl(masked_l2) - 1;
587
588 /* process port */
589 port = (l1i * LONG_BIT) + l2i;
590 synch_clear_bit(port, &s->evtchn_pending[0]);
591
592 isrc = xen_intr_port_to_isrc[port];
593 if (__predict_false(isrc == NULL))
594 continue;
595
596 /* Make sure we are firing on the right vCPU */
597 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
598 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
599 PCPU_GET(cpuid), isrc->xi_cpu));
600
601 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
602
603 /*
604 * If this is the final port processed,
605 * we'll pick up here+1 next time.
606 */
607 pc->last_processed_l1i = l1i;
608 pc->last_processed_l2i = l2i;
609
610 } while (l2i != LONG_BIT - 1);
611
612 l2 = xen_intr_active_ports(pc, s, l1i);
613 if (l2 == 0) {
614 /*
615 * We handled all ports, so we can clear the
616 * selector bit.
617 */
618 l1 &= ~(1UL << l1i);
619 }
620 }
621
622 if (xen_evtchn_needs_ack)
623 lapic_eoi();
624
625 critical_exit();
626 }
627
628 static int
xen_intr_init(void * dummy __unused)629 xen_intr_init(void *dummy __unused)
630 {
631 shared_info_t *s = HYPERVISOR_shared_info;
632 struct xen_intr_pcpu_data *pcpu;
633 struct physdev_pirq_eoi_gmfn eoi_gmfn;
634 int i, rc;
635
636 if (!xen_domain())
637 return (0);
638
639 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
640
641 /*
642 * Set the per-cpu mask of CPU#0 to enable all, since by default all
643 * event channels are bound to CPU#0.
644 */
645 CPU_FOREACH(i) {
646 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
647 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
648 sizeof(pcpu->evtchn_enabled));
649 }
650
651 for (i = 0; i < nitems(s->evtchn_mask); i++)
652 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
653
654 /* Try to register PIRQ EOI map */
655 xen_intr_pirq_eoi_map = malloc(PAGE_SIZE, M_XENINTR, M_WAITOK | M_ZERO);
656 eoi_gmfn.gmfn = atop(vtophys(xen_intr_pirq_eoi_map));
657 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
658 if (rc != 0 && bootverbose)
659 printf("Xen interrupts: unable to register PIRQ EOI map\n");
660 else
661 xen_intr_pirq_eoi_map_enabled = true;
662
663 intr_register_pic(&xen_intr_pic);
664 if (xen_pv_domain() && xen_initial_domain())
665 intr_register_pic(&xen_intr_pirq_pic);
666
667 if (bootverbose)
668 printf("Xen interrupt system initialized\n");
669
670 return (0);
671 }
672 SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
673
674 static void
xen_intrcnt_init(void * dummy __unused)675 xen_intrcnt_init(void *dummy __unused)
676 {
677 unsigned int i;
678
679 if (!xen_domain())
680 return;
681
682 /*
683 * Register interrupt count manually as we aren't guaranteed to see a
684 * call to xen_intr_assign_cpu() before our first interrupt.
685 */
686 CPU_FOREACH(i)
687 xen_intr_intrcnt_add(i);
688 }
689 SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL);
690
691 void
xen_intr_alloc_irqs(void)692 xen_intr_alloc_irqs(void)
693 {
694
695 if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS)
696 panic("IRQ allocation overflow (num_msi_irqs too high?)");
697 first_evtchn_irq = num_io_irqs;
698 num_io_irqs += NR_EVENT_CHANNELS;
699 }
700
701 /*--------------------------- Common PIC Functions ---------------------------*/
702 /**
703 * Prepare this PIC for system suspension.
704 */
705 static void
xen_intr_suspend(struct pic * unused)706 xen_intr_suspend(struct pic *unused)
707 {
708 }
709
710 static void
xen_rebind_ipi(struct xenisrc * isrc)711 xen_rebind_ipi(struct xenisrc *isrc)
712 {
713 #ifdef SMP
714 int cpu = isrc->xi_cpu;
715 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
716 int error;
717 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
718
719 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
720 &bind_ipi);
721 if (error != 0)
722 panic("unable to rebind xen IPI: %d", error);
723
724 isrc->xi_port = bind_ipi.port;
725 isrc->xi_cpu = 0;
726 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
727
728 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
729 cpu_apic_ids[cpu]);
730 if (error)
731 panic("unable to bind xen IPI to CPU#%d: %d",
732 cpu, error);
733
734 evtchn_unmask_port(bind_ipi.port);
735 #else
736 panic("Resume IPI event channel on UP");
737 #endif
738 }
739
740 static void
xen_rebind_virq(struct xenisrc * isrc)741 xen_rebind_virq(struct xenisrc *isrc)
742 {
743 int cpu = isrc->xi_cpu;
744 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
745 int error;
746 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
747 .vcpu = vcpu_id };
748
749 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
750 &bind_virq);
751 if (error != 0)
752 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
753
754 isrc->xi_port = bind_virq.port;
755 isrc->xi_cpu = 0;
756 xen_intr_port_to_isrc[bind_virq.port] = isrc;
757
758 #ifdef SMP
759 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
760 cpu_apic_ids[cpu]);
761 if (error)
762 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
763 isrc->xi_virq, cpu, error);
764 #endif
765
766 evtchn_unmask_port(bind_virq.port);
767 }
768
769 /**
770 * Return this PIC to service after being suspended.
771 */
772 static void
xen_intr_resume(struct pic * unused,bool suspend_cancelled)773 xen_intr_resume(struct pic *unused, bool suspend_cancelled)
774 {
775 shared_info_t *s = HYPERVISOR_shared_info;
776 struct xenisrc *isrc;
777 u_int isrc_idx;
778 int i;
779
780 if (suspend_cancelled)
781 return;
782
783 /* Reset the per-CPU masks */
784 CPU_FOREACH(i) {
785 struct xen_intr_pcpu_data *pcpu;
786
787 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
788 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
789 sizeof(pcpu->evtchn_enabled));
790 }
791
792 /* Mask all event channels. */
793 for (i = 0; i < nitems(s->evtchn_mask); i++)
794 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
795
796 /* Remove port -> isrc mappings */
797 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
798
799 /* Free unused isrcs and rebind VIRQs and IPIs */
800 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) {
801 u_int vector;
802
803 vector = first_evtchn_irq + isrc_idx;
804 isrc = (struct xenisrc *)intr_lookup_source(vector);
805 if (isrc != NULL) {
806 isrc->xi_port = 0;
807 switch (isrc->xi_type) {
808 case EVTCHN_TYPE_IPI:
809 xen_rebind_ipi(isrc);
810 break;
811 case EVTCHN_TYPE_VIRQ:
812 xen_rebind_virq(isrc);
813 break;
814 default:
815 break;
816 }
817 }
818 }
819 }
820
821 /**
822 * Disable a Xen interrupt source.
823 *
824 * \param isrc The interrupt source to disable.
825 */
826 static void
xen_intr_disable_intr(struct intsrc * base_isrc)827 xen_intr_disable_intr(struct intsrc *base_isrc)
828 {
829 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
830
831 evtchn_mask_port(isrc->xi_port);
832 }
833
834 /**
835 * Determine the global interrupt vector number for
836 * a Xen interrupt source.
837 *
838 * \param isrc The interrupt source to query.
839 *
840 * \return The vector number corresponding to the given interrupt source.
841 */
842 static int
xen_intr_vector(struct intsrc * base_isrc)843 xen_intr_vector(struct intsrc *base_isrc)
844 {
845 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
846
847 return (isrc->xi_vector);
848 }
849
850 /**
851 * Determine whether or not interrupt events are pending on the
852 * the given interrupt source.
853 *
854 * \param isrc The interrupt source to query.
855 *
856 * \returns 0 if no events are pending, otherwise non-zero.
857 */
858 static int
xen_intr_source_pending(struct intsrc * isrc)859 xen_intr_source_pending(struct intsrc *isrc)
860 {
861 /*
862 * EventChannels are edge triggered and never masked.
863 * There can be no pending events.
864 */
865 return (0);
866 }
867
868 /**
869 * Perform configuration of an interrupt source.
870 *
871 * \param isrc The interrupt source to configure.
872 * \param trig Edge or level.
873 * \param pol Active high or low.
874 *
875 * \returns 0 if no events are pending, otherwise non-zero.
876 */
877 static int
xen_intr_config_intr(struct intsrc * isrc,enum intr_trigger trig,enum intr_polarity pol)878 xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
879 enum intr_polarity pol)
880 {
881 /* Configuration is only possible via the evtchn apis. */
882 return (ENODEV);
883 }
884
885 /**
886 * Configure CPU affinity for interrupt source event delivery.
887 *
888 * \param isrc The interrupt source to configure.
889 * \param apic_id The apic id of the CPU for handling future events.
890 *
891 * \returns 0 if successful, otherwise an errno.
892 */
893 static int
xen_intr_assign_cpu(struct intsrc * base_isrc,u_int apic_id)894 xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
895 {
896 #ifdef SMP
897 struct evtchn_bind_vcpu bind_vcpu;
898 struct xenisrc *isrc;
899 u_int to_cpu, vcpu_id;
900 int error, masked;
901
902 if (xen_vector_callback_enabled == 0)
903 return (EOPNOTSUPP);
904
905 to_cpu = apic_cpuid(apic_id);
906 vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id;
907
908 mtx_lock(&xen_intr_isrc_lock);
909 isrc = (struct xenisrc *)base_isrc;
910 if (!is_valid_evtchn(isrc->xi_port)) {
911 mtx_unlock(&xen_intr_isrc_lock);
912 return (EINVAL);
913 }
914
915 /*
916 * Mask the event channel while binding it to prevent interrupt
917 * delivery with an inconsistent state in isrc->xi_cpu.
918 */
919 masked = evtchn_test_and_set_mask(isrc->xi_port);
920 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
921 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
922 /*
923 * Virtual IRQs are associated with a cpu by
924 * the Hypervisor at evtchn_bind_virq time, so
925 * all we need to do is update the per-CPU masks.
926 */
927 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
928 isrc->xi_cpu = to_cpu;
929 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
930 goto out;
931 }
932
933 bind_vcpu.port = isrc->xi_port;
934 bind_vcpu.vcpu = vcpu_id;
935
936 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
937 if (isrc->xi_cpu != to_cpu) {
938 if (error == 0) {
939 /* Commit to new binding by removing the old one. */
940 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
941 isrc->xi_cpu = to_cpu;
942 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
943 }
944 }
945
946 out:
947 if (masked == 0)
948 evtchn_unmask_port(isrc->xi_port);
949 mtx_unlock(&xen_intr_isrc_lock);
950 return (0);
951 #else
952 return (EOPNOTSUPP);
953 #endif
954 }
955
956 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
957 /*
958 * Mask a level triggered interrupt source.
959 *
960 * \param isrc The interrupt source to mask (if necessary).
961 * \param eoi If non-zero, perform any necessary end-of-interrupt
962 * acknowledgements.
963 */
964 static void
xen_intr_disable_source(struct intsrc * base_isrc,int eoi)965 xen_intr_disable_source(struct intsrc *base_isrc, int eoi)
966 {
967 struct xenisrc *isrc;
968
969 isrc = (struct xenisrc *)base_isrc;
970
971 /*
972 * NB: checking if the event channel is already masked is
973 * needed because the event channel user-space device
974 * masks event channels on its filter as part of its
975 * normal operation, and those shouldn't be automatically
976 * unmasked by the generic interrupt code. The event channel
977 * device will unmask them when needed.
978 */
979 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
980 }
981
982 /*
983 * Unmask a level triggered interrupt source.
984 *
985 * \param isrc The interrupt source to unmask (if necessary).
986 */
987 static void
xen_intr_enable_source(struct intsrc * base_isrc)988 xen_intr_enable_source(struct intsrc *base_isrc)
989 {
990 struct xenisrc *isrc;
991
992 isrc = (struct xenisrc *)base_isrc;
993
994 if (isrc->xi_masked == 0)
995 evtchn_unmask_port(isrc->xi_port);
996 }
997
998 /*
999 * Perform any necessary end-of-interrupt acknowledgements.
1000 *
1001 * \param isrc The interrupt source to EOI.
1002 */
1003 static void
xen_intr_eoi_source(struct intsrc * base_isrc)1004 xen_intr_eoi_source(struct intsrc *base_isrc)
1005 {
1006 }
1007
1008 /*
1009 * Enable and unmask the interrupt source.
1010 *
1011 * \param isrc The interrupt source to enable.
1012 */
1013 static void
xen_intr_enable_intr(struct intsrc * base_isrc)1014 xen_intr_enable_intr(struct intsrc *base_isrc)
1015 {
1016 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1017
1018 evtchn_unmask_port(isrc->xi_port);
1019 }
1020
1021 /*------------------ Physical Interrupt Source PIC Functions -----------------*/
1022 /*
1023 * Mask a level triggered interrupt source.
1024 *
1025 * \param isrc The interrupt source to mask (if necessary).
1026 * \param eoi If non-zero, perform any necessary end-of-interrupt
1027 * acknowledgements.
1028 */
1029 static void
xen_intr_pirq_disable_source(struct intsrc * base_isrc,int eoi)1030 xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi)
1031 {
1032 struct xenisrc *isrc;
1033
1034 isrc = (struct xenisrc *)base_isrc;
1035
1036 if (isrc->xi_edgetrigger == 0)
1037 evtchn_mask_port(isrc->xi_port);
1038 if (eoi == PIC_EOI)
1039 xen_intr_pirq_eoi_source(base_isrc);
1040 }
1041
1042 /*
1043 * Unmask a level triggered interrupt source.
1044 *
1045 * \param isrc The interrupt source to unmask (if necessary).
1046 */
1047 static void
xen_intr_pirq_enable_source(struct intsrc * base_isrc)1048 xen_intr_pirq_enable_source(struct intsrc *base_isrc)
1049 {
1050 struct xenisrc *isrc;
1051
1052 isrc = (struct xenisrc *)base_isrc;
1053
1054 if (isrc->xi_edgetrigger == 0)
1055 evtchn_unmask_port(isrc->xi_port);
1056 }
1057
1058 /*
1059 * Perform any necessary end-of-interrupt acknowledgements.
1060 *
1061 * \param isrc The interrupt source to EOI.
1062 */
1063 static void
xen_intr_pirq_eoi_source(struct intsrc * base_isrc)1064 xen_intr_pirq_eoi_source(struct intsrc *base_isrc)
1065 {
1066 struct xenisrc *isrc;
1067 int error;
1068
1069 isrc = (struct xenisrc *)base_isrc;
1070
1071 if (xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map)) {
1072 struct physdev_eoi eoi = { .irq = isrc->xi_pirq };
1073
1074 error = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
1075 if (error != 0)
1076 panic("Unable to EOI PIRQ#%d: %d\n",
1077 isrc->xi_pirq, error);
1078 }
1079 }
1080
1081 /*
1082 * Enable and unmask the interrupt source.
1083 *
1084 * \param isrc The interrupt source to enable.
1085 */
1086 static void
xen_intr_pirq_enable_intr(struct intsrc * base_isrc)1087 xen_intr_pirq_enable_intr(struct intsrc *base_isrc)
1088 {
1089 struct xenisrc *isrc;
1090 struct evtchn_bind_pirq bind_pirq;
1091 struct physdev_irq_status_query irq_status;
1092 int error;
1093
1094 isrc = (struct xenisrc *)base_isrc;
1095
1096 if (!xen_intr_pirq_eoi_map_enabled) {
1097 irq_status.irq = isrc->xi_pirq;
1098 error = HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query,
1099 &irq_status);
1100 if (error)
1101 panic("unable to get status of IRQ#%d", isrc->xi_pirq);
1102
1103 if (irq_status.flags & XENIRQSTAT_needs_eoi) {
1104 /*
1105 * Since the dynamic PIRQ EOI map is not available
1106 * mark the PIRQ as needing EOI unconditionally.
1107 */
1108 xen_set_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map);
1109 }
1110 }
1111
1112 bind_pirq.pirq = isrc->xi_pirq;
1113 bind_pirq.flags = isrc->xi_edgetrigger ? 0 : BIND_PIRQ__WILL_SHARE;
1114 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
1115 if (error)
1116 panic("unable to bind IRQ#%d", isrc->xi_pirq);
1117
1118 isrc->xi_port = bind_pirq.port;
1119
1120 mtx_lock(&xen_intr_isrc_lock);
1121 KASSERT((xen_intr_port_to_isrc[bind_pirq.port] == NULL),
1122 ("trying to override an already setup event channel port"));
1123 xen_intr_port_to_isrc[bind_pirq.port] = isrc;
1124 mtx_unlock(&xen_intr_isrc_lock);
1125
1126 evtchn_unmask_port(isrc->xi_port);
1127 }
1128
1129 /*
1130 * Disable an interrupt source.
1131 *
1132 * \param isrc The interrupt source to disable.
1133 */
1134 static void
xen_intr_pirq_disable_intr(struct intsrc * base_isrc)1135 xen_intr_pirq_disable_intr(struct intsrc *base_isrc)
1136 {
1137 struct xenisrc *isrc;
1138 struct evtchn_close close;
1139 int error;
1140
1141 isrc = (struct xenisrc *)base_isrc;
1142
1143 evtchn_mask_port(isrc->xi_port);
1144
1145 close.port = isrc->xi_port;
1146 error = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
1147 if (error)
1148 panic("unable to close event channel %d IRQ#%d",
1149 isrc->xi_port, isrc->xi_pirq);
1150
1151 mtx_lock(&xen_intr_isrc_lock);
1152 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
1153 mtx_unlock(&xen_intr_isrc_lock);
1154
1155 isrc->xi_port = 0;
1156 }
1157
1158 /**
1159 * Perform configuration of an interrupt source.
1160 *
1161 * \param isrc The interrupt source to configure.
1162 * \param trig Edge or level.
1163 * \param pol Active high or low.
1164 *
1165 * \returns 0 if no events are pending, otherwise non-zero.
1166 */
1167 static int
xen_intr_pirq_config_intr(struct intsrc * base_isrc,enum intr_trigger trig,enum intr_polarity pol)1168 xen_intr_pirq_config_intr(struct intsrc *base_isrc, enum intr_trigger trig,
1169 enum intr_polarity pol)
1170 {
1171 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1172 struct physdev_setup_gsi setup_gsi;
1173 int error;
1174
1175 KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM),
1176 ("%s: Conforming trigger or polarity\n", __func__));
1177
1178 setup_gsi.gsi = isrc->xi_pirq;
1179 setup_gsi.triggering = trig == INTR_TRIGGER_EDGE ? 0 : 1;
1180 setup_gsi.polarity = pol == INTR_POLARITY_HIGH ? 0 : 1;
1181
1182 error = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi);
1183 if (error == -XEN_EEXIST) {
1184 if ((isrc->xi_edgetrigger && (trig != INTR_TRIGGER_EDGE)) ||
1185 (isrc->xi_activehi && (pol != INTR_POLARITY_HIGH)))
1186 panic("unable to reconfigure interrupt IRQ#%d",
1187 isrc->xi_pirq);
1188 error = 0;
1189 }
1190 if (error)
1191 panic("unable to configure IRQ#%d\n", isrc->xi_pirq);
1192
1193 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1194 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1195
1196 return (0);
1197 }
1198
1199 /*--------------------------- Public Functions -------------------------------*/
1200 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
1201 int
xen_intr_bind_local_port(device_t dev,evtchn_port_t local_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1202 xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
1203 driver_filter_t filter, driver_intr_t handler, void *arg,
1204 enum intr_type flags, xen_intr_handle_t *port_handlep)
1205 {
1206 struct xenisrc *isrc;
1207 int error;
1208
1209 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT,
1210 device_get_nameunit(dev), filter, handler, arg, flags,
1211 port_handlep);
1212 if (error != 0)
1213 return (error);
1214
1215 /*
1216 * The Event Channel API didn't open this port, so it is not
1217 * responsible for closing it automatically on unbind.
1218 */
1219 isrc->xi_close = 0;
1220 return (0);
1221 }
1222
1223 int
xen_intr_alloc_and_bind_local_port(device_t dev,u_int remote_domain,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1224 xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
1225 driver_filter_t filter, driver_intr_t handler, void *arg,
1226 enum intr_type flags, xen_intr_handle_t *port_handlep)
1227 {
1228 struct xenisrc *isrc;
1229 struct evtchn_alloc_unbound alloc_unbound;
1230 int error;
1231
1232 alloc_unbound.dom = DOMID_SELF;
1233 alloc_unbound.remote_dom = remote_domain;
1234 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
1235 &alloc_unbound);
1236 if (error != 0) {
1237 /*
1238 * XXX Trap Hypercall error code Linuxisms in
1239 * the HYPERCALL layer.
1240 */
1241 return (-error);
1242 }
1243
1244 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1245 device_get_nameunit(dev), filter, handler, arg, flags,
1246 port_handlep);
1247 if (error != 0) {
1248 evtchn_close_t close = { .port = alloc_unbound.port };
1249 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1250 panic("EVTCHNOP_close failed");
1251 return (error);
1252 }
1253
1254 isrc->xi_close = 1;
1255 return (0);
1256 }
1257
1258 int
xen_intr_bind_remote_port(device_t dev,u_int remote_domain,u_int remote_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1259 xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1260 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1261 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1262 {
1263 struct xenisrc *isrc;
1264 struct evtchn_bind_interdomain bind_interdomain;
1265 int error;
1266
1267 bind_interdomain.remote_dom = remote_domain;
1268 bind_interdomain.remote_port = remote_port;
1269 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1270 &bind_interdomain);
1271 if (error != 0) {
1272 /*
1273 * XXX Trap Hypercall error code Linuxisms in
1274 * the HYPERCALL layer.
1275 */
1276 return (-error);
1277 }
1278
1279 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1280 EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg,
1281 flags, port_handlep);
1282 if (error) {
1283 evtchn_close_t close = { .port = bind_interdomain.local_port };
1284 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1285 panic("EVTCHNOP_close failed");
1286 return (error);
1287 }
1288
1289 /*
1290 * The Event Channel API opened this port, so it is
1291 * responsible for closing it automatically on unbind.
1292 */
1293 isrc->xi_close = 1;
1294 return (0);
1295 }
1296
1297 int
xen_intr_bind_virq(device_t dev,u_int virq,u_int cpu,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1298 xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1299 driver_filter_t filter, driver_intr_t handler, void *arg,
1300 enum intr_type flags, xen_intr_handle_t *port_handlep)
1301 {
1302 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1303 struct xenisrc *isrc;
1304 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
1305 int error;
1306
1307 isrc = NULL;
1308 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1309 if (error != 0) {
1310 /*
1311 * XXX Trap Hypercall error code Linuxisms in
1312 * the HYPERCALL layer.
1313 */
1314 return (-error);
1315 }
1316
1317 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ,
1318 device_get_nameunit(dev), filter, handler, arg, flags,
1319 port_handlep);
1320
1321 #ifdef SMP
1322 if (error == 0)
1323 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1324 #endif
1325
1326 if (error != 0) {
1327 evtchn_close_t close = { .port = bind_virq.port };
1328
1329 xen_intr_unbind(*port_handlep);
1330 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1331 panic("EVTCHNOP_close failed");
1332 return (error);
1333 }
1334
1335 #ifdef SMP
1336 if (isrc->xi_cpu != cpu) {
1337 /*
1338 * Too early in the boot process for the generic interrupt
1339 * code to perform the binding. Update our event channel
1340 * masks manually so events can't fire on the wrong cpu
1341 * during AP startup.
1342 */
1343 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1344 }
1345 #endif
1346
1347 /*
1348 * The Event Channel API opened this port, so it is
1349 * responsible for closing it automatically on unbind.
1350 */
1351 isrc->xi_close = 1;
1352 isrc->xi_virq = virq;
1353
1354 return (0);
1355 }
1356
1357 int
xen_intr_alloc_and_bind_ipi(u_int cpu,driver_filter_t filter,enum intr_type flags,xen_intr_handle_t * port_handlep)1358 xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter,
1359 enum intr_type flags, xen_intr_handle_t *port_handlep)
1360 {
1361 #ifdef SMP
1362 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1363 struct xenisrc *isrc;
1364 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
1365 /* Same size as the one used by intr_handler->ih_name. */
1366 char name[MAXCOMLEN + 1];
1367 int error;
1368
1369 isrc = NULL;
1370 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1371 if (error != 0) {
1372 /*
1373 * XXX Trap Hypercall error code Linuxisms in
1374 * the HYPERCALL layer.
1375 */
1376 return (-error);
1377 }
1378
1379 snprintf(name, sizeof(name), "cpu%u", cpu);
1380
1381 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1382 name, filter, NULL, NULL, flags, port_handlep);
1383 if (error != 0) {
1384 evtchn_close_t close = { .port = bind_ipi.port };
1385
1386 xen_intr_unbind(*port_handlep);
1387 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1388 panic("EVTCHNOP_close failed");
1389 return (error);
1390 }
1391
1392 if (isrc->xi_cpu != cpu) {
1393 /*
1394 * Too early in the boot process for the generic interrupt
1395 * code to perform the binding. Update our event channel
1396 * masks manually so events can't fire on the wrong cpu
1397 * during AP startup.
1398 */
1399 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1400 }
1401
1402 /*
1403 * The Event Channel API opened this port, so it is
1404 * responsible for closing it automatically on unbind.
1405 */
1406 isrc->xi_close = 1;
1407 return (0);
1408 #else
1409 return (EOPNOTSUPP);
1410 #endif
1411 }
1412
1413 int
xen_register_pirq(int vector,enum intr_trigger trig,enum intr_polarity pol)1414 xen_register_pirq(int vector, enum intr_trigger trig, enum intr_polarity pol)
1415 {
1416 struct physdev_map_pirq map_pirq;
1417 struct xenisrc *isrc;
1418 int error;
1419
1420 if (vector == 0)
1421 return (EINVAL);
1422
1423 if (bootverbose)
1424 printf("xen: register IRQ#%d\n", vector);
1425
1426 map_pirq.domid = DOMID_SELF;
1427 map_pirq.type = MAP_PIRQ_TYPE_GSI;
1428 map_pirq.index = vector;
1429 map_pirq.pirq = vector;
1430
1431 error = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_pirq);
1432 if (error) {
1433 printf("xen: unable to map IRQ#%d\n", vector);
1434 return (error);
1435 }
1436
1437 mtx_lock(&xen_intr_isrc_lock);
1438 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector);
1439 mtx_unlock(&xen_intr_isrc_lock);
1440 KASSERT((isrc != NULL), ("xen: unable to allocate isrc for interrupt"));
1441 isrc->xi_pirq = vector;
1442 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1443 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1444
1445 return (0);
1446 }
1447
1448 int
xen_register_msi(device_t dev,int vector,int count)1449 xen_register_msi(device_t dev, int vector, int count)
1450 {
1451 struct physdev_map_pirq msi_irq;
1452 struct xenisrc *isrc;
1453 int ret;
1454
1455 memset(&msi_irq, 0, sizeof(msi_irq));
1456 msi_irq.domid = DOMID_SELF;
1457 msi_irq.type = count == 1 ?
1458 MAP_PIRQ_TYPE_MSI_SEG : MAP_PIRQ_TYPE_MULTI_MSI;
1459 msi_irq.index = -1;
1460 msi_irq.pirq = -1;
1461 msi_irq.bus = pci_get_bus(dev) | (pci_get_domain(dev) << 16);
1462 msi_irq.devfn = (pci_get_slot(dev) << 3) | pci_get_function(dev);
1463 msi_irq.entry_nr = count;
1464
1465 ret = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &msi_irq);
1466 if (ret != 0)
1467 return (ret);
1468 if (count != msi_irq.entry_nr) {
1469 panic("unable to setup all requested MSI vectors "
1470 "(expected %d got %d)", count, msi_irq.entry_nr);
1471 }
1472
1473 mtx_lock(&xen_intr_isrc_lock);
1474 for (int i = 0; i < count; i++) {
1475 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector + i);
1476 KASSERT(isrc != NULL,
1477 ("xen: unable to allocate isrc for interrupt"));
1478 isrc->xi_pirq = msi_irq.pirq + i;
1479 /* MSI interrupts are always edge triggered */
1480 isrc->xi_edgetrigger = 1;
1481 }
1482 mtx_unlock(&xen_intr_isrc_lock);
1483
1484 return (0);
1485 }
1486
1487 int
xen_release_msi(int vector)1488 xen_release_msi(int vector)
1489 {
1490 struct physdev_unmap_pirq unmap;
1491 struct xenisrc *isrc;
1492 int ret;
1493
1494 isrc = (struct xenisrc *)intr_lookup_source(vector);
1495 if (isrc == NULL)
1496 return (ENXIO);
1497
1498 unmap.pirq = isrc->xi_pirq;
1499 ret = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap);
1500 if (ret != 0)
1501 return (ret);
1502
1503 xen_intr_release_isrc(isrc);
1504
1505 return (0);
1506 }
1507
1508 int
xen_intr_describe(xen_intr_handle_t port_handle,const char * fmt,...)1509 xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1510 {
1511 char descr[MAXCOMLEN + 1];
1512 struct xenisrc *isrc;
1513 va_list ap;
1514
1515 isrc = xen_intr_isrc(port_handle);
1516 if (isrc == NULL)
1517 return (EINVAL);
1518
1519 va_start(ap, fmt);
1520 vsnprintf(descr, sizeof(descr), fmt, ap);
1521 va_end(ap);
1522 return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr));
1523 }
1524
1525 void
xen_intr_unbind(xen_intr_handle_t * port_handlep)1526 xen_intr_unbind(xen_intr_handle_t *port_handlep)
1527 {
1528 struct xenisrc *isrc;
1529
1530 KASSERT(port_handlep != NULL,
1531 ("NULL xen_intr_handle_t passed to xen_intr_unbind"));
1532
1533 isrc = xen_intr_isrc(*port_handlep);
1534 *port_handlep = NULL;
1535 if (isrc == NULL)
1536 return;
1537
1538 mtx_lock(&xen_intr_isrc_lock);
1539 if (refcount_release(&isrc->xi_refcount) == 0) {
1540 mtx_unlock(&xen_intr_isrc_lock);
1541 return;
1542 }
1543 mtx_unlock(&xen_intr_isrc_lock);
1544
1545 if (isrc->xi_cookie != NULL)
1546 intr_remove_handler(isrc->xi_cookie);
1547 xen_intr_release_isrc(isrc);
1548 }
1549
1550 void
xen_intr_signal(xen_intr_handle_t handle)1551 xen_intr_signal(xen_intr_handle_t handle)
1552 {
1553 struct xenisrc *isrc;
1554
1555 isrc = xen_intr_isrc(handle);
1556 if (isrc != NULL) {
1557 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1558 isrc->xi_type == EVTCHN_TYPE_IPI,
1559 ("evtchn_signal on something other than a local port"));
1560 struct evtchn_send send = { .port = isrc->xi_port };
1561 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1562 }
1563 }
1564
1565 evtchn_port_t
xen_intr_port(xen_intr_handle_t handle)1566 xen_intr_port(xen_intr_handle_t handle)
1567 {
1568 struct xenisrc *isrc;
1569
1570 isrc = xen_intr_isrc(handle);
1571 if (isrc == NULL)
1572 return (0);
1573
1574 return (isrc->xi_port);
1575 }
1576
1577 int
xen_intr_add_handler(const char * name,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t handle)1578 xen_intr_add_handler(const char *name, driver_filter_t filter,
1579 driver_intr_t handler, void *arg, enum intr_type flags,
1580 xen_intr_handle_t handle)
1581 {
1582 struct xenisrc *isrc;
1583 int error;
1584
1585 isrc = xen_intr_isrc(handle);
1586 if (isrc == NULL || isrc->xi_cookie != NULL)
1587 return (EINVAL);
1588
1589 error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg,
1590 flags|INTR_EXCL, &isrc->xi_cookie, 0);
1591 if (error != 0) {
1592 printf(
1593 "%s: xen_intr_add_handler: intr_add_handler failed: %d\n",
1594 name, error);
1595 }
1596
1597 return (error);
1598 }
1599
1600 int
xen_intr_get_evtchn_from_port(evtchn_port_t port,xen_intr_handle_t * handlep)1601 xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep)
1602 {
1603
1604 if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS)
1605 return (EINVAL);
1606
1607 if (handlep == NULL) {
1608 return (EINVAL);
1609 }
1610
1611 mtx_lock(&xen_intr_isrc_lock);
1612 if (xen_intr_port_to_isrc[port] == NULL) {
1613 mtx_unlock(&xen_intr_isrc_lock);
1614 return (EINVAL);
1615 }
1616 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount);
1617 mtx_unlock(&xen_intr_isrc_lock);
1618
1619 /* Assign the opaque handler (the event channel port) */
1620 *handlep = &xen_intr_port_to_isrc[port]->xi_vector;
1621
1622 return (0);
1623 }
1624
1625 #ifdef DDB
1626 static const char *
xen_intr_print_type(enum evtchn_type type)1627 xen_intr_print_type(enum evtchn_type type)
1628 {
1629 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
1630 [EVTCHN_TYPE_UNBOUND] = "UNBOUND",
1631 [EVTCHN_TYPE_PIRQ] = "PIRQ",
1632 [EVTCHN_TYPE_VIRQ] = "VIRQ",
1633 [EVTCHN_TYPE_IPI] = "IPI",
1634 [EVTCHN_TYPE_PORT] = "PORT",
1635 };
1636
1637 if (type >= EVTCHN_TYPE_COUNT)
1638 return ("UNKNOWN");
1639
1640 return (evtchn_type_to_string[type]);
1641 }
1642
1643 static void
xen_intr_dump_port(struct xenisrc * isrc)1644 xen_intr_dump_port(struct xenisrc *isrc)
1645 {
1646 struct xen_intr_pcpu_data *pcpu;
1647 shared_info_t *s = HYPERVISOR_shared_info;
1648 int i;
1649
1650 db_printf("Port %d Type: %s\n",
1651 isrc->xi_port, xen_intr_print_type(isrc->xi_type));
1652 if (isrc->xi_type == EVTCHN_TYPE_PIRQ) {
1653 db_printf("\tPirq: %d ActiveHi: %d EdgeTrigger: %d "
1654 "NeedsEOI: %d\n",
1655 isrc->xi_pirq, isrc->xi_activehi, isrc->xi_edgetrigger,
1656 !!xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map));
1657 }
1658 if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
1659 db_printf("\tVirq: %d\n", isrc->xi_virq);
1660
1661 db_printf("\tMasked: %d Pending: %d\n",
1662 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
1663 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
1664
1665 db_printf("\tPer-CPU Masks: ");
1666 CPU_FOREACH(i) {
1667 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
1668 db_printf("cpu#%d: %d ", i,
1669 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
1670 }
1671 db_printf("\n");
1672 }
1673
DB_SHOW_COMMAND(xen_evtchn,db_show_xen_evtchn)1674 DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
1675 {
1676 int i;
1677
1678 if (!xen_domain()) {
1679 db_printf("Only available on Xen guests\n");
1680 return;
1681 }
1682
1683 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1684 struct xenisrc *isrc;
1685
1686 isrc = xen_intr_port_to_isrc[i];
1687 if (isrc == NULL)
1688 continue;
1689
1690 xen_intr_dump_port(isrc);
1691 }
1692 }
1693 #endif /* DDB */
1694