1 /*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/param.h>
33 #include <sys/lock.h>
34 #include <sys/kernel.h>
35 #include <sys/malloc.h>
36 #include <sys/mutex.h>
37 #include <sys/systm.h>
38 #include <sys/smp.h>
39
40 #include <x86/specialreg.h>
41 #include <x86/apicreg.h>
42
43 #include <machine/clock.h>
44 #include <machine/smp.h>
45
46 #include <machine/vmm.h>
47
48 #include "vmm_lapic.h"
49 #include "vmm_ktr.h"
50 #include "vmm_stat.h"
51
52 #include "vlapic.h"
53 #include "vlapic_priv.h"
54 #include "vioapic.h"
55
56 #define PRIO(x) ((x) >> 4)
57
58 #define VLAPIC_VERSION (16)
59
60 #define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
61
62 /*
63 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
64 * vlapic_callout_handler() and vcpu accesses to:
65 * - timer_freq_bt, timer_period_bt, timer_fire_bt
66 * - timer LVT register
67 */
68 #define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
69 #define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
70 #define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
71
72 /*
73 * APIC timer frequency:
74 * - arbitrary but chosen to be in the ballpark of contemporary hardware.
75 * - power-of-two to avoid loss of precision when converted to a bintime.
76 */
77 #define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
78
79 static __inline uint32_t
vlapic_get_id(struct vlapic * vlapic)80 vlapic_get_id(struct vlapic *vlapic)
81 {
82
83 if (x2apic(vlapic))
84 return (vlapic->vcpuid);
85 else
86 return (vlapic->vcpuid << 24);
87 }
88
89 static uint32_t
x2apic_ldr(struct vlapic * vlapic)90 x2apic_ldr(struct vlapic *vlapic)
91 {
92 int apicid;
93 uint32_t ldr;
94
95 apicid = vlapic_get_id(vlapic);
96 ldr = 1 << (apicid & 0xf);
97 ldr |= (apicid & 0xffff0) << 12;
98 return (ldr);
99 }
100
101 void
vlapic_dfr_write_handler(struct vlapic * vlapic)102 vlapic_dfr_write_handler(struct vlapic *vlapic)
103 {
104 struct LAPIC *lapic;
105
106 lapic = vlapic->apic_page;
107 if (x2apic(vlapic)) {
108 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
109 lapic->dfr);
110 lapic->dfr = 0;
111 return;
112 }
113
114 lapic->dfr &= APIC_DFR_MODEL_MASK;
115 lapic->dfr |= APIC_DFR_RESERVED;
116
117 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
118 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
119 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
120 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
121 else
122 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
123 }
124
125 void
vlapic_ldr_write_handler(struct vlapic * vlapic)126 vlapic_ldr_write_handler(struct vlapic *vlapic)
127 {
128 struct LAPIC *lapic;
129
130 lapic = vlapic->apic_page;
131
132 /* LDR is read-only in x2apic mode */
133 if (x2apic(vlapic)) {
134 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
135 lapic->ldr);
136 lapic->ldr = x2apic_ldr(vlapic);
137 } else {
138 lapic->ldr &= ~APIC_LDR_RESERVED;
139 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
140 }
141 }
142
143 void
vlapic_id_write_handler(struct vlapic * vlapic)144 vlapic_id_write_handler(struct vlapic *vlapic)
145 {
146 struct LAPIC *lapic;
147
148 /*
149 * We don't allow the ID register to be modified so reset it back to
150 * its default value.
151 */
152 lapic = vlapic->apic_page;
153 lapic->id = vlapic_get_id(vlapic);
154 }
155
156 static int
vlapic_timer_divisor(uint32_t dcr)157 vlapic_timer_divisor(uint32_t dcr)
158 {
159 switch (dcr & 0xB) {
160 case APIC_TDCR_1:
161 return (1);
162 case APIC_TDCR_2:
163 return (2);
164 case APIC_TDCR_4:
165 return (4);
166 case APIC_TDCR_8:
167 return (8);
168 case APIC_TDCR_16:
169 return (16);
170 case APIC_TDCR_32:
171 return (32);
172 case APIC_TDCR_64:
173 return (64);
174 case APIC_TDCR_128:
175 return (128);
176 default:
177 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
178 }
179 }
180
181 #if 0
182 static inline void
183 vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
184 {
185 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
186 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
187 *lvt & APIC_LVTT_M);
188 }
189 #endif
190
191 static uint32_t
vlapic_get_ccr(struct vlapic * vlapic)192 vlapic_get_ccr(struct vlapic *vlapic)
193 {
194 struct bintime bt_now, bt_rem;
195 struct LAPIC *lapic;
196 uint32_t ccr;
197
198 ccr = 0;
199 lapic = vlapic->apic_page;
200
201 VLAPIC_TIMER_LOCK(vlapic);
202 if (callout_active(&vlapic->callout)) {
203 /*
204 * If the timer is scheduled to expire in the future then
205 * compute the value of 'ccr' based on the remaining time.
206 */
207 binuptime(&bt_now);
208 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
209 bt_rem = vlapic->timer_fire_bt;
210 bintime_sub(&bt_rem, &bt_now);
211 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
212 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
213 }
214 }
215 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
216 "icr_timer is %#x", ccr, lapic->icr_timer));
217 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
218 ccr, lapic->icr_timer);
219 VLAPIC_TIMER_UNLOCK(vlapic);
220 return (ccr);
221 }
222
223 void
vlapic_dcr_write_handler(struct vlapic * vlapic)224 vlapic_dcr_write_handler(struct vlapic *vlapic)
225 {
226 struct LAPIC *lapic;
227 int divisor;
228
229 lapic = vlapic->apic_page;
230 VLAPIC_TIMER_LOCK(vlapic);
231
232 divisor = vlapic_timer_divisor(lapic->dcr_timer);
233 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
234 lapic->dcr_timer, divisor);
235
236 /*
237 * Update the timer frequency and the timer period.
238 *
239 * XXX changes to the frequency divider will not take effect until
240 * the timer is reloaded.
241 */
242 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
243 vlapic->timer_period_bt = vlapic->timer_freq_bt;
244 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
245
246 VLAPIC_TIMER_UNLOCK(vlapic);
247 }
248
249 void
vlapic_esr_write_handler(struct vlapic * vlapic)250 vlapic_esr_write_handler(struct vlapic *vlapic)
251 {
252 struct LAPIC *lapic;
253
254 lapic = vlapic->apic_page;
255 lapic->esr = vlapic->esr_pending;
256 vlapic->esr_pending = 0;
257 }
258
259 int
vlapic_set_intr_ready(struct vlapic * vlapic,int vector,bool level)260 vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
261 {
262 struct LAPIC *lapic;
263 uint32_t *irrptr, *tmrptr, mask;
264 int idx;
265
266 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
267
268 lapic = vlapic->apic_page;
269 if (!(lapic->svr & APIC_SVR_ENABLE)) {
270 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
271 "interrupt %d", vector);
272 return (0);
273 }
274
275 if (vector < 16) {
276 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
277 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
278 vector);
279 return (1);
280 }
281
282 if (vlapic->ops.set_intr_ready)
283 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
284
285 idx = (vector / 32) * 4;
286 mask = 1 << (vector % 32);
287
288 irrptr = &lapic->irr0;
289 atomic_set_int(&irrptr[idx], mask);
290
291 /*
292 * Verify that the trigger-mode of the interrupt matches with
293 * the vlapic TMR registers.
294 */
295 tmrptr = &lapic->tmr0;
296 if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
297 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
298 "interrupt is %s-triggered", idx / 4, tmrptr[idx],
299 level ? "level" : "edge");
300 }
301
302 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
303 return (1);
304 }
305
306 static __inline uint32_t *
vlapic_get_lvtptr(struct vlapic * vlapic,uint32_t offset)307 vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
308 {
309 struct LAPIC *lapic = vlapic->apic_page;
310 int i;
311
312 switch (offset) {
313 case APIC_OFFSET_CMCI_LVT:
314 return (&lapic->lvt_cmci);
315 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
316 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
317 return ((&lapic->lvt_timer) + i);
318 default:
319 panic("vlapic_get_lvt: invalid LVT\n");
320 }
321 }
322
323 static __inline int
lvt_off_to_idx(uint32_t offset)324 lvt_off_to_idx(uint32_t offset)
325 {
326 int index;
327
328 switch (offset) {
329 case APIC_OFFSET_CMCI_LVT:
330 index = APIC_LVT_CMCI;
331 break;
332 case APIC_OFFSET_TIMER_LVT:
333 index = APIC_LVT_TIMER;
334 break;
335 case APIC_OFFSET_THERM_LVT:
336 index = APIC_LVT_THERMAL;
337 break;
338 case APIC_OFFSET_PERF_LVT:
339 index = APIC_LVT_PMC;
340 break;
341 case APIC_OFFSET_LINT0_LVT:
342 index = APIC_LVT_LINT0;
343 break;
344 case APIC_OFFSET_LINT1_LVT:
345 index = APIC_LVT_LINT1;
346 break;
347 case APIC_OFFSET_ERROR_LVT:
348 index = APIC_LVT_ERROR;
349 break;
350 default:
351 index = -1;
352 break;
353 }
354 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
355 "invalid lvt index %d for offset %#x", index, offset));
356
357 return (index);
358 }
359
360 static __inline uint32_t
vlapic_get_lvt(struct vlapic * vlapic,uint32_t offset)361 vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
362 {
363 int idx;
364 uint32_t val;
365
366 idx = lvt_off_to_idx(offset);
367 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
368 return (val);
369 }
370
371 void
vlapic_lvt_write_handler(struct vlapic * vlapic,uint32_t offset)372 vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
373 {
374 uint32_t *lvtptr, mask, val;
375 struct LAPIC *lapic;
376 int idx;
377
378 lapic = vlapic->apic_page;
379 lvtptr = vlapic_get_lvtptr(vlapic, offset);
380 val = *lvtptr;
381 idx = lvt_off_to_idx(offset);
382
383 if (!(lapic->svr & APIC_SVR_ENABLE))
384 val |= APIC_LVT_M;
385 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
386 switch (offset) {
387 case APIC_OFFSET_TIMER_LVT:
388 mask |= APIC_LVTT_TM;
389 break;
390 case APIC_OFFSET_ERROR_LVT:
391 break;
392 case APIC_OFFSET_LINT0_LVT:
393 case APIC_OFFSET_LINT1_LVT:
394 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
395 /* FALLTHROUGH */
396 default:
397 mask |= APIC_LVT_DM;
398 break;
399 }
400 val &= mask;
401 *lvtptr = val;
402 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
403 }
404
405 static void
vlapic_mask_lvts(struct vlapic * vlapic)406 vlapic_mask_lvts(struct vlapic *vlapic)
407 {
408 struct LAPIC *lapic = vlapic->apic_page;
409
410 lapic->lvt_cmci |= APIC_LVT_M;
411 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
412
413 lapic->lvt_timer |= APIC_LVT_M;
414 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
415
416 lapic->lvt_thermal |= APIC_LVT_M;
417 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
418
419 lapic->lvt_pcint |= APIC_LVT_M;
420 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
421
422 lapic->lvt_lint0 |= APIC_LVT_M;
423 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
424
425 lapic->lvt_lint1 |= APIC_LVT_M;
426 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
427
428 lapic->lvt_error |= APIC_LVT_M;
429 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
430 }
431
432 static int
vlapic_fire_lvt(struct vlapic * vlapic,uint32_t lvt)433 vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
434 {
435 uint32_t vec, mode;
436
437 if (lvt & APIC_LVT_M)
438 return (0);
439
440 vec = lvt & APIC_LVT_VECTOR;
441 mode = lvt & APIC_LVT_DM;
442
443 switch (mode) {
444 case APIC_LVT_DM_FIXED:
445 if (vec < 16) {
446 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
447 return (0);
448 }
449 if (vlapic_set_intr_ready(vlapic, vec, false))
450 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
451 break;
452 case APIC_LVT_DM_NMI:
453 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
454 break;
455 case APIC_LVT_DM_EXTINT:
456 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
457 break;
458 default:
459 // Other modes ignored
460 return (0);
461 }
462 return (1);
463 }
464
465 #if 1
466 static void
dump_isrvec_stk(struct vlapic * vlapic)467 dump_isrvec_stk(struct vlapic *vlapic)
468 {
469 int i;
470 uint32_t *isrptr;
471
472 isrptr = &vlapic->apic_page->isr0;
473 for (i = 0; i < 8; i++)
474 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
475
476 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
477 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
478 }
479 #endif
480
481 /*
482 * Algorithm adopted from section "Interrupt, Task and Processor Priority"
483 * in Intel Architecture Manual Vol 3a.
484 */
485 static void
vlapic_update_ppr(struct vlapic * vlapic)486 vlapic_update_ppr(struct vlapic *vlapic)
487 {
488 int isrvec, tpr, ppr;
489
490 /*
491 * Note that the value on the stack at index 0 is always 0.
492 *
493 * This is a placeholder for the value of ISRV when none of the
494 * bits is set in the ISRx registers.
495 */
496 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
497 tpr = vlapic->apic_page->tpr;
498
499 #if 1
500 {
501 int i, lastprio, curprio, vector, idx;
502 uint32_t *isrptr;
503
504 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
505 panic("isrvec_stk is corrupted: %d", isrvec);
506
507 /*
508 * Make sure that the priority of the nested interrupts is
509 * always increasing.
510 */
511 lastprio = -1;
512 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
513 curprio = PRIO(vlapic->isrvec_stk[i]);
514 if (curprio <= lastprio) {
515 dump_isrvec_stk(vlapic);
516 panic("isrvec_stk does not satisfy invariant");
517 }
518 lastprio = curprio;
519 }
520
521 /*
522 * Make sure that each bit set in the ISRx registers has a
523 * corresponding entry on the isrvec stack.
524 */
525 i = 1;
526 isrptr = &vlapic->apic_page->isr0;
527 for (vector = 0; vector < 256; vector++) {
528 idx = (vector / 32) * 4;
529 if (isrptr[idx] & (1 << (vector % 32))) {
530 if (i > vlapic->isrvec_stk_top ||
531 vlapic->isrvec_stk[i] != vector) {
532 dump_isrvec_stk(vlapic);
533 panic("ISR and isrvec_stk out of sync");
534 }
535 i++;
536 }
537 }
538 }
539 #endif
540
541 if (PRIO(tpr) >= PRIO(isrvec))
542 ppr = tpr;
543 else
544 ppr = isrvec & 0xf0;
545
546 vlapic->apic_page->ppr = ppr;
547 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
548 }
549
550 void
vlapic_sync_tpr(struct vlapic * vlapic)551 vlapic_sync_tpr(struct vlapic *vlapic)
552 {
553 vlapic_update_ppr(vlapic);
554 }
555
556 static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
557
558 static void
vlapic_process_eoi(struct vlapic * vlapic)559 vlapic_process_eoi(struct vlapic *vlapic)
560 {
561 struct LAPIC *lapic = vlapic->apic_page;
562 uint32_t *isrptr, *tmrptr;
563 int i, idx, bitpos, vector;
564
565 isrptr = &lapic->isr0;
566 tmrptr = &lapic->tmr0;
567
568 for (i = 7; i >= 0; i--) {
569 idx = i * 4;
570 bitpos = fls(isrptr[idx]);
571 if (bitpos-- != 0) {
572 if (vlapic->isrvec_stk_top <= 0) {
573 panic("invalid vlapic isrvec_stk_top %d",
574 vlapic->isrvec_stk_top);
575 }
576 isrptr[idx] &= ~(1 << bitpos);
577 vector = i * 32 + bitpos;
578 VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d",
579 vector);
580 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
581 vlapic->isrvec_stk_top--;
582 vlapic_update_ppr(vlapic);
583 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
584 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
585 vector);
586 }
587 return;
588 }
589 }
590 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI");
591 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1);
592 }
593
594 static __inline int
vlapic_get_lvt_field(uint32_t lvt,uint32_t mask)595 vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
596 {
597
598 return (lvt & mask);
599 }
600
601 static __inline int
vlapic_periodic_timer(struct vlapic * vlapic)602 vlapic_periodic_timer(struct vlapic *vlapic)
603 {
604 uint32_t lvt;
605
606 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
607
608 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
609 }
610
611 static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
612
613 void
vlapic_set_error(struct vlapic * vlapic,uint32_t mask)614 vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
615 {
616 uint32_t lvt;
617
618 vlapic->esr_pending |= mask;
619 if (vlapic->esr_firing)
620 return;
621 vlapic->esr_firing = 1;
622
623 // The error LVT always uses the fixed delivery mode.
624 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
625 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
626 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
627 }
628 vlapic->esr_firing = 0;
629 }
630
631 static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
632
633 static void
vlapic_fire_timer(struct vlapic * vlapic)634 vlapic_fire_timer(struct vlapic *vlapic)
635 {
636 uint32_t lvt;
637
638 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
639
640 // The timer LVT always uses the fixed delivery mode.
641 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
642 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
643 VLAPIC_CTR0(vlapic, "vlapic timer fired");
644 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
645 }
646 }
647
648 static VMM_STAT(VLAPIC_INTR_CMC,
649 "corrected machine check interrupts generated by vlapic");
650
651 void
vlapic_fire_cmci(struct vlapic * vlapic)652 vlapic_fire_cmci(struct vlapic *vlapic)
653 {
654 uint32_t lvt;
655
656 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
657 if (vlapic_fire_lvt(vlapic, lvt)) {
658 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
659 }
660 }
661
662 static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
663 "lvts triggered");
664
665 int
vlapic_trigger_lvt(struct vlapic * vlapic,int vector)666 vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
667 {
668 uint32_t lvt;
669
670 if (vlapic_enabled(vlapic) == false) {
671 /*
672 * When the local APIC is global/hardware disabled,
673 * LINT[1:0] pins are configured as INTR and NMI pins,
674 * respectively.
675 */
676 switch (vector) {
677 case APIC_LVT_LINT0:
678 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
679 break;
680 case APIC_LVT_LINT1:
681 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
682 break;
683 default:
684 break;
685 }
686 return (0);
687 }
688
689 switch (vector) {
690 case APIC_LVT_LINT0:
691 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
692 break;
693 case APIC_LVT_LINT1:
694 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
695 break;
696 case APIC_LVT_TIMER:
697 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
698 lvt |= APIC_LVT_DM_FIXED;
699 break;
700 case APIC_LVT_ERROR:
701 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
702 lvt |= APIC_LVT_DM_FIXED;
703 break;
704 case APIC_LVT_PMC:
705 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
706 break;
707 case APIC_LVT_THERMAL:
708 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
709 break;
710 case APIC_LVT_CMCI:
711 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
712 break;
713 default:
714 return (EINVAL);
715 }
716 if (vlapic_fire_lvt(vlapic, lvt)) {
717 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
718 LVTS_TRIGGERRED, vector, 1);
719 }
720 return (0);
721 }
722
723 static void
vlapic_callout_handler(void * arg)724 vlapic_callout_handler(void *arg)
725 {
726 struct vlapic *vlapic;
727 struct bintime bt, btnow;
728 sbintime_t rem_sbt;
729
730 vlapic = arg;
731
732 VLAPIC_TIMER_LOCK(vlapic);
733 if (callout_pending(&vlapic->callout)) /* callout was reset */
734 goto done;
735
736 if (!callout_active(&vlapic->callout)) /* callout was stopped */
737 goto done;
738
739 callout_deactivate(&vlapic->callout);
740
741 vlapic_fire_timer(vlapic);
742
743 if (vlapic_periodic_timer(vlapic)) {
744 binuptime(&btnow);
745 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
746 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
747 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
748 vlapic->timer_fire_bt.frac));
749
750 /*
751 * Compute the delta between when the timer was supposed to
752 * fire and the present time.
753 */
754 bt = btnow;
755 bintime_sub(&bt, &vlapic->timer_fire_bt);
756
757 rem_sbt = bttosbt(vlapic->timer_period_bt);
758 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
759 /*
760 * Adjust the time until the next countdown downward
761 * to account for the lost time.
762 */
763 rem_sbt -= bttosbt(bt);
764 } else {
765 /*
766 * If the delta is greater than the timer period then
767 * just reset our time base instead of trying to catch
768 * up.
769 */
770 vlapic->timer_fire_bt = btnow;
771 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
772 "usecs, period is %lu usecs - resetting time base",
773 bttosbt(bt) / SBT_1US,
774 bttosbt(vlapic->timer_period_bt) / SBT_1US);
775 }
776
777 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
778 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
779 vlapic_callout_handler, vlapic, 0);
780 }
781 done:
782 VLAPIC_TIMER_UNLOCK(vlapic);
783 }
784
785 void
vlapic_icrtmr_write_handler(struct vlapic * vlapic)786 vlapic_icrtmr_write_handler(struct vlapic *vlapic)
787 {
788 struct LAPIC *lapic;
789 sbintime_t sbt;
790 uint32_t icr_timer;
791
792 VLAPIC_TIMER_LOCK(vlapic);
793
794 lapic = vlapic->apic_page;
795 icr_timer = lapic->icr_timer;
796
797 vlapic->timer_period_bt = vlapic->timer_freq_bt;
798 bintime_mul(&vlapic->timer_period_bt, icr_timer);
799
800 if (icr_timer != 0) {
801 binuptime(&vlapic->timer_fire_bt);
802 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
803
804 sbt = bttosbt(vlapic->timer_period_bt);
805 callout_reset_sbt(&vlapic->callout, sbt, 0,
806 vlapic_callout_handler, vlapic, 0);
807 } else
808 callout_stop(&vlapic->callout);
809
810 VLAPIC_TIMER_UNLOCK(vlapic);
811 }
812
813 /*
814 * This function populates 'dmask' with the set of vcpus that match the
815 * addressing specified by the (dest, phys, lowprio) tuple.
816 *
817 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
818 * or xAPIC (8-bit) destination field.
819 */
820 static void
vlapic_calcdest(struct vm * vm,cpuset_t * dmask,uint32_t dest,bool phys,bool lowprio,bool x2apic_dest)821 vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
822 bool lowprio, bool x2apic_dest)
823 {
824 struct vlapic *vlapic;
825 uint32_t dfr, ldr, ldest, cluster;
826 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
827 cpuset_t amask;
828 int vcpuid;
829
830 if ((x2apic_dest && dest == 0xffffffff) ||
831 (!x2apic_dest && dest == 0xff)) {
832 /*
833 * Broadcast in both logical and physical modes.
834 */
835 *dmask = vm_active_cpus(vm);
836 return;
837 }
838
839 if (phys) {
840 /*
841 * Physical mode: destination is APIC ID.
842 */
843 CPU_ZERO(dmask);
844 vcpuid = vm_apicid2vcpuid(vm, dest);
845 amask = vm_active_cpus(vm);
846 if (vcpuid < vm_get_maxcpus(vm) && CPU_ISSET(vcpuid, &amask))
847 CPU_SET(vcpuid, dmask);
848 } else {
849 /*
850 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
851 * bitmask. This model is only available in the xAPIC mode.
852 */
853 mda_flat_ldest = dest & 0xff;
854
855 /*
856 * In the "Cluster Model" the MDA is used to identify a
857 * specific cluster and a set of APICs in that cluster.
858 */
859 if (x2apic_dest) {
860 mda_cluster_id = dest >> 16;
861 mda_cluster_ldest = dest & 0xffff;
862 } else {
863 mda_cluster_id = (dest >> 4) & 0xf;
864 mda_cluster_ldest = dest & 0xf;
865 }
866
867 /*
868 * Logical mode: match each APIC that has a bit set
869 * in it's LDR that matches a bit in the ldest.
870 */
871 CPU_ZERO(dmask);
872 amask = vm_active_cpus(vm);
873 while ((vcpuid = CPU_FFS(&amask)) != 0) {
874 vcpuid--;
875 CPU_CLR(vcpuid, &amask);
876
877 vlapic = vm_lapic(vm, vcpuid);
878 dfr = vlapic->apic_page->dfr;
879 ldr = vlapic->apic_page->ldr;
880
881 if ((dfr & APIC_DFR_MODEL_MASK) ==
882 APIC_DFR_MODEL_FLAT) {
883 ldest = ldr >> 24;
884 mda_ldest = mda_flat_ldest;
885 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
886 APIC_DFR_MODEL_CLUSTER) {
887 if (x2apic(vlapic)) {
888 cluster = ldr >> 16;
889 ldest = ldr & 0xffff;
890 } else {
891 cluster = ldr >> 28;
892 ldest = (ldr >> 24) & 0xf;
893 }
894 if (cluster != mda_cluster_id)
895 continue;
896 mda_ldest = mda_cluster_ldest;
897 } else {
898 /*
899 * Guest has configured a bad logical
900 * model for this vcpu - skip it.
901 */
902 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
903 "model %x - cannot deliver interrupt", dfr);
904 continue;
905 }
906
907 if ((mda_ldest & ldest) != 0) {
908 CPU_SET(vcpuid, dmask);
909 if (lowprio)
910 break;
911 }
912 }
913 }
914 }
915
916 static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
917
918 static void
vlapic_set_tpr(struct vlapic * vlapic,uint8_t val)919 vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
920 {
921 struct LAPIC *lapic = vlapic->apic_page;
922
923 if (lapic->tpr != val) {
924 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed "
925 "from %#x to %#x", lapic->tpr, val);
926 lapic->tpr = val;
927 vlapic_update_ppr(vlapic);
928 }
929 }
930
931 static uint8_t
vlapic_get_tpr(struct vlapic * vlapic)932 vlapic_get_tpr(struct vlapic *vlapic)
933 {
934 struct LAPIC *lapic = vlapic->apic_page;
935
936 return (lapic->tpr);
937 }
938
939 void
vlapic_set_cr8(struct vlapic * vlapic,uint64_t val)940 vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
941 {
942 uint8_t tpr;
943
944 if (val & ~0xf) {
945 vm_inject_gp(vlapic->vm, vlapic->vcpuid);
946 return;
947 }
948
949 tpr = val << 4;
950 vlapic_set_tpr(vlapic, tpr);
951 }
952
953 uint64_t
vlapic_get_cr8(struct vlapic * vlapic)954 vlapic_get_cr8(struct vlapic *vlapic)
955 {
956 uint8_t tpr;
957
958 tpr = vlapic_get_tpr(vlapic);
959 return (tpr >> 4);
960 }
961
962 int
vlapic_icrlo_write_handler(struct vlapic * vlapic,bool * retu)963 vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
964 {
965 int i;
966 bool phys;
967 cpuset_t dmask;
968 uint64_t icrval;
969 uint32_t dest, vec, mode;
970 struct vlapic *vlapic2;
971 struct vm_exit *vmexit;
972 struct LAPIC *lapic;
973 uint16_t maxcpus;
974
975 lapic = vlapic->apic_page;
976 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
977 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
978
979 if (x2apic(vlapic))
980 dest = icrval >> 32;
981 else
982 dest = icrval >> (32 + 24);
983 vec = icrval & APIC_VECTOR_MASK;
984 mode = icrval & APIC_DELMODE_MASK;
985
986 if (mode == APIC_DELMODE_FIXED && vec < 16) {
987 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
988 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
989 return (0);
990 }
991
992 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
993
994 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
995 switch (icrval & APIC_DEST_MASK) {
996 case APIC_DEST_DESTFLD:
997 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
998 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
999 x2apic(vlapic));
1000 break;
1001 case APIC_DEST_SELF:
1002 CPU_SETOF(vlapic->vcpuid, &dmask);
1003 break;
1004 case APIC_DEST_ALLISELF:
1005 dmask = vm_active_cpus(vlapic->vm);
1006 break;
1007 case APIC_DEST_ALLESELF:
1008 dmask = vm_active_cpus(vlapic->vm);
1009 CPU_CLR(vlapic->vcpuid, &dmask);
1010 break;
1011 default:
1012 CPU_ZERO(&dmask); /* satisfy gcc */
1013 break;
1014 }
1015
1016 while ((i = CPU_FFS(&dmask)) != 0) {
1017 i--;
1018 CPU_CLR(i, &dmask);
1019 if (mode == APIC_DELMODE_FIXED) {
1020 lapic_intr_edge(vlapic->vm, i, vec);
1021 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
1022 IPIS_SENT, i, 1);
1023 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
1024 "to vcpuid %d", vec, i);
1025 } else {
1026 vm_inject_nmi(vlapic->vm, i);
1027 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
1028 "to vcpuid %d", i);
1029 }
1030 }
1031
1032 return (0); /* handled completely in the kernel */
1033 }
1034
1035 maxcpus = vm_get_maxcpus(vlapic->vm);
1036 if (mode == APIC_DELMODE_INIT) {
1037 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
1038 return (0);
1039
1040 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1041 vlapic2 = vm_lapic(vlapic->vm, dest);
1042
1043 /* move from INIT to waiting-for-SIPI state */
1044 if (vlapic2->boot_state == BS_INIT) {
1045 vlapic2->boot_state = BS_SIPI;
1046 }
1047
1048 return (0);
1049 }
1050 }
1051
1052 if (mode == APIC_DELMODE_STARTUP) {
1053 if (vlapic->vcpuid == 0 && dest != 0 && dest < maxcpus) {
1054 vlapic2 = vm_lapic(vlapic->vm, dest);
1055
1056 /*
1057 * Ignore SIPIs in any state other than wait-for-SIPI
1058 */
1059 if (vlapic2->boot_state != BS_SIPI)
1060 return (0);
1061
1062 vlapic2->boot_state = BS_RUNNING;
1063
1064 *retu = true;
1065 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1066 vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1067 vmexit->u.spinup_ap.vcpu = dest;
1068 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1069
1070 return (0);
1071 }
1072 }
1073
1074 /*
1075 * This will cause a return to userland.
1076 */
1077 return (1);
1078 }
1079
1080 void
vlapic_self_ipi_handler(struct vlapic * vlapic,uint64_t val)1081 vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1082 {
1083 int vec;
1084
1085 KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1086
1087 vec = val & 0xff;
1088 lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1089 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1090 vlapic->vcpuid, 1);
1091 VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1092 }
1093
1094 int
vlapic_pending_intr(struct vlapic * vlapic,int * vecptr)1095 vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1096 {
1097 struct LAPIC *lapic = vlapic->apic_page;
1098 int idx, i, bitpos, vector;
1099 uint32_t *irrptr, val;
1100
1101 vlapic_update_ppr(vlapic);
1102
1103 if (vlapic->ops.pending_intr)
1104 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1105
1106 irrptr = &lapic->irr0;
1107
1108 for (i = 7; i >= 0; i--) {
1109 idx = i * 4;
1110 val = atomic_load_acq_int(&irrptr[idx]);
1111 bitpos = fls(val);
1112 if (bitpos != 0) {
1113 vector = i * 32 + (bitpos - 1);
1114 if (PRIO(vector) > PRIO(lapic->ppr)) {
1115 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1116 if (vecptr != NULL)
1117 *vecptr = vector;
1118 return (1);
1119 } else
1120 break;
1121 }
1122 }
1123 return (0);
1124 }
1125
1126 void
vlapic_intr_accepted(struct vlapic * vlapic,int vector)1127 vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1128 {
1129 struct LAPIC *lapic = vlapic->apic_page;
1130 uint32_t *irrptr, *isrptr;
1131 int idx, stk_top;
1132
1133 if (vlapic->ops.intr_accepted)
1134 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1135
1136 /*
1137 * clear the ready bit for vector being accepted in irr
1138 * and set the vector as in service in isr.
1139 */
1140 idx = (vector / 32) * 4;
1141
1142 irrptr = &lapic->irr0;
1143 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1144 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1145
1146 isrptr = &lapic->isr0;
1147 isrptr[idx] |= 1 << (vector % 32);
1148 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1149
1150 /*
1151 * Update the PPR
1152 */
1153 vlapic->isrvec_stk_top++;
1154
1155 stk_top = vlapic->isrvec_stk_top;
1156 if (stk_top >= ISRVEC_STK_SIZE)
1157 panic("isrvec_stk_top overflow %d", stk_top);
1158
1159 vlapic->isrvec_stk[stk_top] = vector;
1160 }
1161
1162 void
vlapic_svr_write_handler(struct vlapic * vlapic)1163 vlapic_svr_write_handler(struct vlapic *vlapic)
1164 {
1165 struct LAPIC *lapic;
1166 uint32_t old, new, changed;
1167
1168 lapic = vlapic->apic_page;
1169
1170 new = lapic->svr;
1171 old = vlapic->svr_last;
1172 vlapic->svr_last = new;
1173
1174 changed = old ^ new;
1175 if ((changed & APIC_SVR_ENABLE) != 0) {
1176 if ((new & APIC_SVR_ENABLE) == 0) {
1177 /*
1178 * The apic is now disabled so stop the apic timer
1179 * and mask all the LVT entries.
1180 */
1181 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1182 VLAPIC_TIMER_LOCK(vlapic);
1183 callout_stop(&vlapic->callout);
1184 VLAPIC_TIMER_UNLOCK(vlapic);
1185 vlapic_mask_lvts(vlapic);
1186 } else {
1187 /*
1188 * The apic is now enabled so restart the apic timer
1189 * if it is configured in periodic mode.
1190 */
1191 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1192 if (vlapic_periodic_timer(vlapic))
1193 vlapic_icrtmr_write_handler(vlapic);
1194 }
1195 }
1196 }
1197
1198 int
vlapic_read(struct vlapic * vlapic,int mmio_access,uint64_t offset,uint64_t * data,bool * retu)1199 vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1200 uint64_t *data, bool *retu)
1201 {
1202 struct LAPIC *lapic = vlapic->apic_page;
1203 uint32_t *reg;
1204 int i;
1205
1206 /* Ignore MMIO accesses in x2APIC mode */
1207 if (x2apic(vlapic) && mmio_access) {
1208 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1209 offset);
1210 *data = 0;
1211 goto done;
1212 }
1213
1214 if (!x2apic(vlapic) && !mmio_access) {
1215 /*
1216 * XXX Generate GP fault for MSR accesses in xAPIC mode
1217 */
1218 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1219 "xAPIC mode", offset);
1220 *data = 0;
1221 goto done;
1222 }
1223
1224 if (offset > sizeof(*lapic)) {
1225 *data = 0;
1226 goto done;
1227 }
1228
1229 offset &= ~3;
1230 switch(offset)
1231 {
1232 case APIC_OFFSET_ID:
1233 *data = lapic->id;
1234 break;
1235 case APIC_OFFSET_VER:
1236 *data = lapic->version;
1237 break;
1238 case APIC_OFFSET_TPR:
1239 *data = vlapic_get_tpr(vlapic);
1240 break;
1241 case APIC_OFFSET_APR:
1242 *data = lapic->apr;
1243 break;
1244 case APIC_OFFSET_PPR:
1245 *data = lapic->ppr;
1246 break;
1247 case APIC_OFFSET_EOI:
1248 *data = lapic->eoi;
1249 break;
1250 case APIC_OFFSET_LDR:
1251 *data = lapic->ldr;
1252 break;
1253 case APIC_OFFSET_DFR:
1254 *data = lapic->dfr;
1255 break;
1256 case APIC_OFFSET_SVR:
1257 *data = lapic->svr;
1258 break;
1259 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1260 i = (offset - APIC_OFFSET_ISR0) >> 2;
1261 reg = &lapic->isr0;
1262 *data = *(reg + i);
1263 break;
1264 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1265 i = (offset - APIC_OFFSET_TMR0) >> 2;
1266 reg = &lapic->tmr0;
1267 *data = *(reg + i);
1268 break;
1269 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1270 i = (offset - APIC_OFFSET_IRR0) >> 2;
1271 reg = &lapic->irr0;
1272 *data = atomic_load_acq_int(reg + i);
1273 break;
1274 case APIC_OFFSET_ESR:
1275 *data = lapic->esr;
1276 break;
1277 case APIC_OFFSET_ICR_LOW:
1278 *data = lapic->icr_lo;
1279 if (x2apic(vlapic))
1280 *data |= (uint64_t)lapic->icr_hi << 32;
1281 break;
1282 case APIC_OFFSET_ICR_HI:
1283 *data = lapic->icr_hi;
1284 break;
1285 case APIC_OFFSET_CMCI_LVT:
1286 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1287 *data = vlapic_get_lvt(vlapic, offset);
1288 #ifdef INVARIANTS
1289 reg = vlapic_get_lvtptr(vlapic, offset);
1290 KASSERT(*data == *reg, ("inconsistent lvt value at "
1291 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1292 #endif
1293 break;
1294 case APIC_OFFSET_TIMER_ICR:
1295 *data = lapic->icr_timer;
1296 break;
1297 case APIC_OFFSET_TIMER_CCR:
1298 *data = vlapic_get_ccr(vlapic);
1299 break;
1300 case APIC_OFFSET_TIMER_DCR:
1301 *data = lapic->dcr_timer;
1302 break;
1303 case APIC_OFFSET_SELF_IPI:
1304 /*
1305 * XXX generate a GP fault if vlapic is in x2apic mode
1306 */
1307 *data = 0;
1308 break;
1309 case APIC_OFFSET_RRR:
1310 default:
1311 *data = 0;
1312 break;
1313 }
1314 done:
1315 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1316 return 0;
1317 }
1318
1319 int
vlapic_write(struct vlapic * vlapic,int mmio_access,uint64_t offset,uint64_t data,bool * retu)1320 vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1321 uint64_t data, bool *retu)
1322 {
1323 struct LAPIC *lapic = vlapic->apic_page;
1324 uint32_t *regptr;
1325 int retval;
1326
1327 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1328 ("vlapic_write: invalid offset %#lx", offset));
1329
1330 VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1331 offset, data);
1332
1333 if (offset > sizeof(*lapic))
1334 return (0);
1335
1336 /* Ignore MMIO accesses in x2APIC mode */
1337 if (x2apic(vlapic) && mmio_access) {
1338 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1339 "in x2APIC mode", data, offset);
1340 return (0);
1341 }
1342
1343 /*
1344 * XXX Generate GP fault for MSR accesses in xAPIC mode
1345 */
1346 if (!x2apic(vlapic) && !mmio_access) {
1347 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1348 "in xAPIC mode", data, offset);
1349 return (0);
1350 }
1351
1352 retval = 0;
1353 switch(offset)
1354 {
1355 case APIC_OFFSET_ID:
1356 lapic->id = data;
1357 vlapic_id_write_handler(vlapic);
1358 break;
1359 case APIC_OFFSET_TPR:
1360 vlapic_set_tpr(vlapic, data & 0xff);
1361 break;
1362 case APIC_OFFSET_EOI:
1363 vlapic_process_eoi(vlapic);
1364 break;
1365 case APIC_OFFSET_LDR:
1366 lapic->ldr = data;
1367 vlapic_ldr_write_handler(vlapic);
1368 break;
1369 case APIC_OFFSET_DFR:
1370 lapic->dfr = data;
1371 vlapic_dfr_write_handler(vlapic);
1372 break;
1373 case APIC_OFFSET_SVR:
1374 lapic->svr = data;
1375 vlapic_svr_write_handler(vlapic);
1376 break;
1377 case APIC_OFFSET_ICR_LOW:
1378 lapic->icr_lo = data;
1379 if (x2apic(vlapic))
1380 lapic->icr_hi = data >> 32;
1381 retval = vlapic_icrlo_write_handler(vlapic, retu);
1382 break;
1383 case APIC_OFFSET_ICR_HI:
1384 lapic->icr_hi = data;
1385 break;
1386 case APIC_OFFSET_CMCI_LVT:
1387 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1388 regptr = vlapic_get_lvtptr(vlapic, offset);
1389 *regptr = data;
1390 vlapic_lvt_write_handler(vlapic, offset);
1391 break;
1392 case APIC_OFFSET_TIMER_ICR:
1393 lapic->icr_timer = data;
1394 vlapic_icrtmr_write_handler(vlapic);
1395 break;
1396
1397 case APIC_OFFSET_TIMER_DCR:
1398 lapic->dcr_timer = data;
1399 vlapic_dcr_write_handler(vlapic);
1400 break;
1401
1402 case APIC_OFFSET_ESR:
1403 vlapic_esr_write_handler(vlapic);
1404 break;
1405
1406 case APIC_OFFSET_SELF_IPI:
1407 if (x2apic(vlapic))
1408 vlapic_self_ipi_handler(vlapic, data);
1409 break;
1410
1411 case APIC_OFFSET_VER:
1412 case APIC_OFFSET_APR:
1413 case APIC_OFFSET_PPR:
1414 case APIC_OFFSET_RRR:
1415 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1416 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1417 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1418 case APIC_OFFSET_TIMER_CCR:
1419 default:
1420 // Read only.
1421 break;
1422 }
1423
1424 return (retval);
1425 }
1426
1427 static void
vlapic_reset(struct vlapic * vlapic)1428 vlapic_reset(struct vlapic *vlapic)
1429 {
1430 struct LAPIC *lapic;
1431
1432 lapic = vlapic->apic_page;
1433 bzero(lapic, sizeof(struct LAPIC));
1434
1435 lapic->id = vlapic_get_id(vlapic);
1436 lapic->version = VLAPIC_VERSION;
1437 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1438 lapic->dfr = 0xffffffff;
1439 lapic->svr = APIC_SVR_VECTOR;
1440 vlapic_mask_lvts(vlapic);
1441 vlapic_reset_tmr(vlapic);
1442
1443 lapic->dcr_timer = 0;
1444 vlapic_dcr_write_handler(vlapic);
1445
1446 if (vlapic->vcpuid == 0)
1447 vlapic->boot_state = BS_RUNNING; /* BSP */
1448 else
1449 vlapic->boot_state = BS_INIT; /* AP */
1450
1451 vlapic->svr_last = lapic->svr;
1452 }
1453
1454 void
vlapic_init(struct vlapic * vlapic)1455 vlapic_init(struct vlapic *vlapic)
1456 {
1457 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1458 KASSERT(vlapic->vcpuid >= 0 &&
1459 vlapic->vcpuid < vm_get_maxcpus(vlapic->vm),
1460 ("vlapic_init: vcpuid is not initialized"));
1461 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1462 "initialized"));
1463
1464 /*
1465 * If the vlapic is configured in x2apic mode then it will be
1466 * accessed in the critical section via the MSR emulation code.
1467 *
1468 * Therefore the timer mutex must be a spinlock because blockable
1469 * mutexes cannot be acquired in a critical section.
1470 */
1471 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1472 callout_init(&vlapic->callout, 1);
1473
1474 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1475
1476 if (vlapic->vcpuid == 0)
1477 vlapic->msr_apicbase |= APICBASE_BSP;
1478
1479 vlapic_reset(vlapic);
1480 }
1481
1482 void
vlapic_cleanup(struct vlapic * vlapic)1483 vlapic_cleanup(struct vlapic *vlapic)
1484 {
1485
1486 callout_drain(&vlapic->callout);
1487 }
1488
1489 uint64_t
vlapic_get_apicbase(struct vlapic * vlapic)1490 vlapic_get_apicbase(struct vlapic *vlapic)
1491 {
1492
1493 return (vlapic->msr_apicbase);
1494 }
1495
1496 int
vlapic_set_apicbase(struct vlapic * vlapic,uint64_t new)1497 vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1498 {
1499
1500 if (vlapic->msr_apicbase != new) {
1501 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1502 "not supported", vlapic->msr_apicbase, new);
1503 return (-1);
1504 }
1505
1506 return (0);
1507 }
1508
1509 void
vlapic_set_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state state)1510 vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1511 {
1512 struct vlapic *vlapic;
1513 struct LAPIC *lapic;
1514
1515 vlapic = vm_lapic(vm, vcpuid);
1516
1517 if (state == X2APIC_DISABLED)
1518 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1519 else
1520 vlapic->msr_apicbase |= APICBASE_X2APIC;
1521
1522 /*
1523 * Reset the local APIC registers whose values are mode-dependent.
1524 *
1525 * XXX this works because the APIC mode can be changed only at vcpu
1526 * initialization time.
1527 */
1528 lapic = vlapic->apic_page;
1529 lapic->id = vlapic_get_id(vlapic);
1530 if (x2apic(vlapic)) {
1531 lapic->ldr = x2apic_ldr(vlapic);
1532 lapic->dfr = 0;
1533 } else {
1534 lapic->ldr = 0;
1535 lapic->dfr = 0xffffffff;
1536 }
1537
1538 if (state == X2APIC_ENABLED) {
1539 if (vlapic->ops.enable_x2apic_mode)
1540 (*vlapic->ops.enable_x2apic_mode)(vlapic);
1541 }
1542 }
1543
1544 void
vlapic_deliver_intr(struct vm * vm,bool level,uint32_t dest,bool phys,int delmode,int vec)1545 vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1546 int delmode, int vec)
1547 {
1548 bool lowprio;
1549 int vcpuid;
1550 cpuset_t dmask;
1551
1552 if (delmode != IOART_DELFIXED &&
1553 delmode != IOART_DELLOPRI &&
1554 delmode != IOART_DELEXINT) {
1555 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1556 return;
1557 }
1558 lowprio = (delmode == IOART_DELLOPRI);
1559
1560 /*
1561 * We don't provide any virtual interrupt redirection hardware so
1562 * all interrupts originating from the ioapic or MSI specify the
1563 * 'dest' in the legacy xAPIC format.
1564 */
1565 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1566
1567 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1568 vcpuid--;
1569 CPU_CLR(vcpuid, &dmask);
1570 if (delmode == IOART_DELEXINT) {
1571 vm_inject_extint(vm, vcpuid);
1572 } else {
1573 lapic_set_intr(vm, vcpuid, vec, level);
1574 }
1575 }
1576 }
1577
1578 void
vlapic_post_intr(struct vlapic * vlapic,int hostcpu,int ipinum)1579 vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1580 {
1581 /*
1582 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1583 *
1584 * This is done by leveraging features like Posted Interrupts (Intel)
1585 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1586 *
1587 * If neither of these features are available then fallback to
1588 * sending an IPI to 'hostcpu'.
1589 */
1590 if (vlapic->ops.post_intr)
1591 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1592 else
1593 ipi_cpu(hostcpu, ipinum);
1594 }
1595
1596 bool
vlapic_enabled(struct vlapic * vlapic)1597 vlapic_enabled(struct vlapic *vlapic)
1598 {
1599 struct LAPIC *lapic = vlapic->apic_page;
1600
1601 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1602 (lapic->svr & APIC_SVR_ENABLE) != 0)
1603 return (true);
1604 else
1605 return (false);
1606 }
1607
1608 static void
vlapic_set_tmr(struct vlapic * vlapic,int vector,bool level)1609 vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1610 {
1611 struct LAPIC *lapic;
1612 uint32_t *tmrptr, mask;
1613 int idx;
1614
1615 lapic = vlapic->apic_page;
1616 tmrptr = &lapic->tmr0;
1617 idx = (vector / 32) * 4;
1618 mask = 1 << (vector % 32);
1619 if (level)
1620 tmrptr[idx] |= mask;
1621 else
1622 tmrptr[idx] &= ~mask;
1623
1624 if (vlapic->ops.set_tmr != NULL)
1625 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1626 }
1627
1628 void
vlapic_reset_tmr(struct vlapic * vlapic)1629 vlapic_reset_tmr(struct vlapic *vlapic)
1630 {
1631 int vector;
1632
1633 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1634
1635 for (vector = 0; vector <= 255; vector++)
1636 vlapic_set_tmr(vlapic, vector, false);
1637 }
1638
1639 void
vlapic_set_tmr_level(struct vlapic * vlapic,uint32_t dest,bool phys,int delmode,int vector)1640 vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1641 int delmode, int vector)
1642 {
1643 cpuset_t dmask;
1644 bool lowprio;
1645
1646 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1647
1648 /*
1649 * A level trigger is valid only for fixed and lowprio delivery modes.
1650 */
1651 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1652 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1653 "delivery-mode %d", delmode);
1654 return;
1655 }
1656
1657 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1658 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1659
1660 if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1661 return;
1662
1663 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1664 vlapic_set_tmr(vlapic, vector, true);
1665 }
1666