1 /*-
2 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
3 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD: stable/10/sys/amd64/vmm/io/vioapic.c 284900 2015-06-28 03:22:26Z neel $
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vioapic.c 284900 2015-06-28 03:22:26Z neel $");
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40
41 #include <x86/apicreg.h>
42 #include <machine/vmm.h>
43
44 #include "vmm_ktr.h"
45 #include "vmm_lapic.h"
46 #include "vlapic.h"
47 #include "vioapic.h"
48
49 #define IOREGSEL 0x00
50 #define IOWIN 0x10
51
52 #define REDIR_ENTRIES 24
53 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
54
55 struct vioapic {
56 struct vm *vm;
57 struct mtx mtx;
58 uint32_t id;
59 uint32_t ioregsel;
60 struct {
61 uint64_t reg;
62 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
63 } rtbl[REDIR_ENTRIES];
64 };
65
66 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
67 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
68 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
69
70 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
71
72 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
73 VM_CTR1((vioapic)->vm, fmt, a1)
74
75 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
76 VM_CTR2((vioapic)->vm, fmt, a1, a2)
77
78 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
79 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
80
81 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
82 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
83
84 #ifdef KTR
85 static const char *
pinstate_str(bool asserted)86 pinstate_str(bool asserted)
87 {
88
89 if (asserted)
90 return ("asserted");
91 else
92 return ("deasserted");
93 }
94 #endif
95
96 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)97 vioapic_send_intr(struct vioapic *vioapic, int pin)
98 {
99 int vector, delmode;
100 uint32_t low, high, dest;
101 bool level, phys;
102
103 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
104 ("vioapic_set_pinstate: invalid pin number %d", pin));
105
106 KASSERT(VIOAPIC_LOCKED(vioapic),
107 ("vioapic_set_pinstate: vioapic is not locked"));
108
109 low = vioapic->rtbl[pin].reg;
110 high = vioapic->rtbl[pin].reg >> 32;
111
112 if ((low & IOART_INTMASK) == IOART_INTMSET) {
113 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
114 return;
115 }
116
117 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
118 delmode = low & IOART_DELMOD;
119 level = low & IOART_TRGRLVL ? true : false;
120 if (level)
121 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
122
123 vector = low & IOART_INTVEC;
124 dest = high >> APIC_ID_SHIFT;
125 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
126 }
127
128 static void
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)129 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
130 {
131 int oldcnt, newcnt;
132 bool needintr;
133
134 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
135 ("vioapic_set_pinstate: invalid pin number %d", pin));
136
137 KASSERT(VIOAPIC_LOCKED(vioapic),
138 ("vioapic_set_pinstate: vioapic is not locked"));
139
140 oldcnt = vioapic->rtbl[pin].acnt;
141 if (newstate)
142 vioapic->rtbl[pin].acnt++;
143 else
144 vioapic->rtbl[pin].acnt--;
145 newcnt = vioapic->rtbl[pin].acnt;
146
147 if (newcnt < 0) {
148 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
149 pin, newcnt);
150 }
151
152 needintr = false;
153 if (oldcnt == 0 && newcnt == 1) {
154 needintr = true;
155 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
156 } else if (oldcnt == 1 && newcnt == 0) {
157 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
158 } else {
159 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
160 pin, pinstate_str(newstate), newcnt);
161 }
162
163 if (needintr)
164 vioapic_send_intr(vioapic, pin);
165 }
166
167 enum irqstate {
168 IRQSTATE_ASSERT,
169 IRQSTATE_DEASSERT,
170 IRQSTATE_PULSE
171 };
172
173 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)174 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
175 {
176 struct vioapic *vioapic;
177
178 if (irq < 0 || irq >= REDIR_ENTRIES)
179 return (EINVAL);
180
181 vioapic = vm_ioapic(vm);
182
183 VIOAPIC_LOCK(vioapic);
184 switch (irqstate) {
185 case IRQSTATE_ASSERT:
186 vioapic_set_pinstate(vioapic, irq, true);
187 break;
188 case IRQSTATE_DEASSERT:
189 vioapic_set_pinstate(vioapic, irq, false);
190 break;
191 case IRQSTATE_PULSE:
192 vioapic_set_pinstate(vioapic, irq, true);
193 vioapic_set_pinstate(vioapic, irq, false);
194 break;
195 default:
196 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
197 }
198 VIOAPIC_UNLOCK(vioapic);
199
200 return (0);
201 }
202
203 int
vioapic_assert_irq(struct vm * vm,int irq)204 vioapic_assert_irq(struct vm *vm, int irq)
205 {
206
207 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
208 }
209
210 int
vioapic_deassert_irq(struct vm * vm,int irq)211 vioapic_deassert_irq(struct vm *vm, int irq)
212 {
213
214 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
215 }
216
217 int
vioapic_pulse_irq(struct vm * vm,int irq)218 vioapic_pulse_irq(struct vm *vm, int irq)
219 {
220
221 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
222 }
223
224 /*
225 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
226 * configuration.
227 */
228 static void
vioapic_update_tmr(struct vm * vm,int vcpuid,void * arg)229 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg)
230 {
231 struct vioapic *vioapic;
232 struct vlapic *vlapic;
233 uint32_t low, high, dest;
234 int delmode, pin, vector;
235 bool level, phys;
236
237 vlapic = vm_lapic(vm, vcpuid);
238 vioapic = vm_ioapic(vm);
239
240 VIOAPIC_LOCK(vioapic);
241 /*
242 * Reset all vectors to be edge-triggered.
243 */
244 vlapic_reset_tmr(vlapic);
245 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
246 low = vioapic->rtbl[pin].reg;
247 high = vioapic->rtbl[pin].reg >> 32;
248
249 level = low & IOART_TRGRLVL ? true : false;
250 if (!level)
251 continue;
252
253 /*
254 * For a level-triggered 'pin' let the vlapic figure out if
255 * an assertion on this 'pin' would result in an interrupt
256 * being delivered to it. If yes, then it will modify the
257 * TMR bit associated with this vector to level-triggered.
258 */
259 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
260 delmode = low & IOART_DELMOD;
261 vector = low & IOART_INTVEC;
262 dest = high >> APIC_ID_SHIFT;
263 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
264 }
265 VIOAPIC_UNLOCK(vioapic);
266 }
267
268 static uint32_t
vioapic_read(struct vioapic * vioapic,int vcpuid,uint32_t addr)269 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
270 {
271 int regnum, pin, rshift;
272
273 regnum = addr & 0xff;
274 switch (regnum) {
275 case IOAPIC_ID:
276 return (vioapic->id);
277 break;
278 case IOAPIC_VER:
279 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
280 break;
281 case IOAPIC_ARB:
282 return (vioapic->id);
283 break;
284 default:
285 break;
286 }
287
288 /* redirection table entries */
289 if (regnum >= IOAPIC_REDTBL &&
290 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
291 pin = (regnum - IOAPIC_REDTBL) / 2;
292 if ((regnum - IOAPIC_REDTBL) % 2)
293 rshift = 32;
294 else
295 rshift = 0;
296
297 return (vioapic->rtbl[pin].reg >> rshift);
298 }
299
300 return (0);
301 }
302
303 static void
vioapic_write(struct vioapic * vioapic,int vcpuid,uint32_t addr,uint32_t data)304 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
305 {
306 uint64_t data64, mask64;
307 uint64_t last, changed;
308 int regnum, pin, lshift;
309 cpuset_t allvcpus;
310
311 regnum = addr & 0xff;
312 switch (regnum) {
313 case IOAPIC_ID:
314 vioapic->id = data & APIC_ID_MASK;
315 break;
316 case IOAPIC_VER:
317 case IOAPIC_ARB:
318 /* readonly */
319 break;
320 default:
321 break;
322 }
323
324 /* redirection table entries */
325 if (regnum >= IOAPIC_REDTBL &&
326 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
327 pin = (regnum - IOAPIC_REDTBL) / 2;
328 if ((regnum - IOAPIC_REDTBL) % 2)
329 lshift = 32;
330 else
331 lshift = 0;
332
333 last = vioapic->rtbl[pin].reg;
334
335 data64 = (uint64_t)data << lshift;
336 mask64 = (uint64_t)0xffffffff << lshift;
337 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
338 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
339
340 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
341 pin, vioapic->rtbl[pin].reg);
342
343 /*
344 * If any fields in the redirection table entry (except mask
345 * or polarity) have changed then rendezvous all the vcpus
346 * to update their vlapic trigger-mode registers.
347 */
348 changed = last ^ vioapic->rtbl[pin].reg;
349 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
350 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
351 "vlapic trigger-mode register", pin);
352 VIOAPIC_UNLOCK(vioapic);
353 allvcpus = vm_active_cpus(vioapic->vm);
354 vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
355 vioapic_update_tmr, NULL);
356 VIOAPIC_LOCK(vioapic);
357 }
358
359 /*
360 * Generate an interrupt if the following conditions are met:
361 * - pin is not masked
362 * - previous interrupt has been EOIed
363 * - pin level is asserted
364 */
365 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR &&
366 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 &&
367 (vioapic->rtbl[pin].acnt > 0)) {
368 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
369 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
370 vioapic_send_intr(vioapic, pin);
371 }
372 }
373 }
374
375 static int
vioapic_mmio_rw(struct vioapic * vioapic,int vcpuid,uint64_t gpa,uint64_t * data,int size,bool doread)376 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
377 uint64_t *data, int size, bool doread)
378 {
379 uint64_t offset;
380
381 offset = gpa - VIOAPIC_BASE;
382
383 /*
384 * The IOAPIC specification allows 32-bit wide accesses to the
385 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
386 */
387 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
388 if (doread)
389 *data = 0;
390 return (0);
391 }
392
393 VIOAPIC_LOCK(vioapic);
394 if (offset == IOREGSEL) {
395 if (doread)
396 *data = vioapic->ioregsel;
397 else
398 vioapic->ioregsel = *data;
399 } else {
400 if (doread) {
401 *data = vioapic_read(vioapic, vcpuid,
402 vioapic->ioregsel);
403 } else {
404 vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
405 *data);
406 }
407 }
408 VIOAPIC_UNLOCK(vioapic);
409
410 return (0);
411 }
412
413 int
vioapic_mmio_read(void * vm,int vcpuid,uint64_t gpa,uint64_t * rval,int size,void * arg)414 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
415 int size, void *arg)
416 {
417 int error;
418 struct vioapic *vioapic;
419
420 vioapic = vm_ioapic(vm);
421 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
422 return (error);
423 }
424
425 int
vioapic_mmio_write(void * vm,int vcpuid,uint64_t gpa,uint64_t wval,int size,void * arg)426 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval,
427 int size, void *arg)
428 {
429 int error;
430 struct vioapic *vioapic;
431
432 vioapic = vm_ioapic(vm);
433 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
434 return (error);
435 }
436
437 void
vioapic_process_eoi(struct vm * vm,int vcpuid,int vector)438 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
439 {
440 struct vioapic *vioapic;
441 int pin;
442
443 KASSERT(vector >= 0 && vector < 256,
444 ("vioapic_process_eoi: invalid vector %d", vector));
445
446 vioapic = vm_ioapic(vm);
447 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
448
449 /*
450 * XXX keep track of the pins associated with this vector instead
451 * of iterating on every single pin each time.
452 */
453 VIOAPIC_LOCK(vioapic);
454 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
455 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
456 continue;
457 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
458 continue;
459 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
460 if (vioapic->rtbl[pin].acnt > 0) {
461 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
462 "acnt %d", pin, vioapic->rtbl[pin].acnt);
463 vioapic_send_intr(vioapic, pin);
464 }
465 }
466 VIOAPIC_UNLOCK(vioapic);
467 }
468
469 struct vioapic *
vioapic_init(struct vm * vm)470 vioapic_init(struct vm *vm)
471 {
472 int i;
473 struct vioapic *vioapic;
474
475 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
476
477 vioapic->vm = vm;
478 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
479
480 /* Initialize all redirection entries to mask all interrupts */
481 for (i = 0; i < REDIR_ENTRIES; i++)
482 vioapic->rtbl[i].reg = 0x0001000000010000UL;
483
484 return (vioapic);
485 }
486
487 void
vioapic_cleanup(struct vioapic * vioapic)488 vioapic_cleanup(struct vioapic *vioapic)
489 {
490
491 free(vioapic, M_VIOAPIC);
492 }
493
494 int
vioapic_pincount(struct vm * vm)495 vioapic_pincount(struct vm *vm)
496 {
497
498 return (REDIR_ENTRIES);
499 }
500