1 /*
2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <sys/param.h>
29 #include <sys/bus.h>
30 #include <sys/kernel.h>
31 #include <sys/malloc.h>
32 #include <sys/proc.h>
33 #include <sys/smp.h>
34 #include <sys/systm.h>
35
36 #include <vm/vm.h>
37 #include <vm/pmap.h>
38
39 #include <machine/cpufunc.h>
40 #include <machine/cpu.h>
41 #include <machine/intr_machdep.h>
42 #include <machine/md_var.h>
43 #include <machine/smp.h>
44
45 #include <x86/apicreg.h>
46 #include <x86/apicvar.h>
47
48 #include <xen/xen-os.h>
49 #include <xen/features.h>
50 #include <xen/gnttab.h>
51 #include <xen/hypervisor.h>
52 #include <xen/hvm.h>
53 #include <xen/xen_intr.h>
54
55 #include <xen/interface/vcpu.h>
56
57 /*--------------------------------- Macros -----------------------------------*/
58
59 #define XEN_APIC_UNSUPPORTED \
60 panic("%s: not available in Xen PV port.", __func__)
61
62 /*--------------------------- Forward Declarations ---------------------------*/
63 #ifdef SMP
64 static driver_filter_t xen_smp_rendezvous_action;
65 #ifdef __amd64__
66 static driver_filter_t xen_invlop;
67 #else
68 static driver_filter_t xen_invltlb;
69 static driver_filter_t xen_invlpg;
70 static driver_filter_t xen_invlrng;
71 static driver_filter_t xen_invlcache;
72 #endif
73 static driver_filter_t xen_ipi_bitmap_handler;
74 static driver_filter_t xen_cpustop_handler;
75 static driver_filter_t xen_cpususpend_handler;
76 static driver_filter_t xen_ipi_swi_handler;
77 #endif
78
79 /*---------------------------------- Macros ----------------------------------*/
80 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
81
82 /*--------------------------------- Xen IPIs ---------------------------------*/
83 #ifdef SMP
84 struct xen_ipi_handler
85 {
86 driver_filter_t *filter;
87 const char *description;
88 };
89
90 static struct xen_ipi_handler xen_ipis[] =
91 {
92 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" },
93 #ifdef __amd64__
94 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"},
95 #else
96 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"},
97 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" },
98 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" },
99 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" },
100 #endif
101 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" },
102 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" },
103 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" },
104 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" },
105 };
106 #endif
107
108 /*
109 * Save previous (native) handler as a fallback. Xen < 4.7 doesn't support
110 * VCPUOP_send_nmi for HVM guests, and thus we need a fallback in that case:
111 *
112 * https://lists.freebsd.org/archives/freebsd-xen/2022-January/000032.html
113 */
114 void (*native_ipi_vectored)(u_int, int);
115
116 /*------------------------------- Per-CPU Data -------------------------------*/
117 #ifdef SMP
118 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
119 #endif
120
121 /*------------------------------- Xen PV APIC --------------------------------*/
122
123 static void
xen_pv_lapic_create(u_int apic_id,int boot_cpu)124 xen_pv_lapic_create(u_int apic_id, int boot_cpu)
125 {
126 #ifdef SMP
127 cpu_add(apic_id, boot_cpu);
128 #endif
129 }
130
131 static void
xen_pv_lapic_init(vm_paddr_t addr)132 xen_pv_lapic_init(vm_paddr_t addr)
133 {
134
135 }
136
137 static void
xen_pv_lapic_setup(int boot)138 xen_pv_lapic_setup(int boot)
139 {
140
141 }
142
143 static void
xen_pv_lapic_dump(const char * str)144 xen_pv_lapic_dump(const char *str)
145 {
146
147 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str);
148 }
149
150 static void
xen_pv_lapic_disable(void)151 xen_pv_lapic_disable(void)
152 {
153
154 }
155
156 static bool
xen_pv_lapic_is_x2apic(void)157 xen_pv_lapic_is_x2apic(void)
158 {
159
160 return (false);
161 }
162
163 static void
xen_pv_lapic_eoi(void)164 xen_pv_lapic_eoi(void)
165 {
166
167 XEN_APIC_UNSUPPORTED;
168 }
169
170 static int
xen_pv_lapic_id(void)171 xen_pv_lapic_id(void)
172 {
173
174 return (PCPU_GET(apic_id));
175 }
176
177 static int
xen_pv_lapic_intr_pending(u_int vector)178 xen_pv_lapic_intr_pending(u_int vector)
179 {
180
181 XEN_APIC_UNSUPPORTED;
182 return (0);
183 }
184
185 static u_int
xen_pv_apic_cpuid(u_int apic_id)186 xen_pv_apic_cpuid(u_int apic_id)
187 {
188 #ifdef SMP
189 return (apic_cpuids[apic_id]);
190 #else
191 return (0);
192 #endif
193 }
194
195 static u_int
xen_pv_apic_alloc_vector(u_int apic_id,u_int irq)196 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq)
197 {
198
199 XEN_APIC_UNSUPPORTED;
200 return (0);
201 }
202
203 static u_int
xen_pv_apic_alloc_vectors(u_int apic_id,u_int * irqs,u_int count,u_int align)204 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
205 {
206
207 XEN_APIC_UNSUPPORTED;
208 return (0);
209 }
210
211 static void
xen_pv_apic_disable_vector(u_int apic_id,u_int vector)212 xen_pv_apic_disable_vector(u_int apic_id, u_int vector)
213 {
214
215 XEN_APIC_UNSUPPORTED;
216 }
217
218 static void
xen_pv_apic_enable_vector(u_int apic_id,u_int vector)219 xen_pv_apic_enable_vector(u_int apic_id, u_int vector)
220 {
221
222 XEN_APIC_UNSUPPORTED;
223 }
224
225 static void
xen_pv_apic_free_vector(u_int apic_id,u_int vector,u_int irq)226 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
227 {
228
229 XEN_APIC_UNSUPPORTED;
230 }
231
232 static void
xen_pv_lapic_calibrate_timer(void)233 xen_pv_lapic_calibrate_timer(void)
234 {
235
236 }
237
238 static void
xen_pv_lapic_set_logical_id(u_int apic_id,u_int cluster,u_int cluster_id)239 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
240 {
241
242 XEN_APIC_UNSUPPORTED;
243 }
244
245 static int
xen_pv_lapic_enable_pmc(void)246 xen_pv_lapic_enable_pmc(void)
247 {
248
249 XEN_APIC_UNSUPPORTED;
250 return (0);
251 }
252
253 static void
xen_pv_lapic_disable_pmc(void)254 xen_pv_lapic_disable_pmc(void)
255 {
256
257 XEN_APIC_UNSUPPORTED;
258 }
259
260 static void
xen_pv_lapic_reenable_pmc(void)261 xen_pv_lapic_reenable_pmc(void)
262 {
263
264 XEN_APIC_UNSUPPORTED;
265 }
266
267 static void
xen_pv_lapic_enable_cmc(void)268 xen_pv_lapic_enable_cmc(void)
269 {
270
271 }
272
273 #ifdef SMP
274 static void
xen_pv_lapic_ipi_raw(register_t icrlo,u_int dest)275 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest)
276 {
277
278 XEN_APIC_UNSUPPORTED;
279 }
280
281 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field)
282 static int
send_nmi(int dest)283 send_nmi(int dest)
284 {
285 unsigned int cpu;
286 int rc = 0;
287
288 /*
289 * NMIs are not routed over event channels, and instead delivered as on
290 * native using the exception vector (#2). Triggering them can be done
291 * using the local APIC, or an hypercall as a shortcut like it's done
292 * below.
293 */
294 switch(dest) {
295 case APIC_IPI_DEST_SELF:
296 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL);
297 break;
298 case APIC_IPI_DEST_ALL:
299 CPU_FOREACH(cpu) {
300 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
301 PCPU_ID_GET(cpu, vcpu_id), NULL);
302 if (rc != 0)
303 break;
304 }
305 break;
306 case APIC_IPI_DEST_OTHERS:
307 CPU_FOREACH(cpu) {
308 if (cpu != PCPU_GET(cpuid)) {
309 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
310 PCPU_ID_GET(cpu, vcpu_id), NULL);
311 if (rc != 0)
312 break;
313 }
314 }
315 break;
316 default:
317 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
318 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL);
319 break;
320 }
321
322 return rc;
323 }
324 #undef PCPU_ID_GET
325
326 static void
xen_pv_lapic_ipi_vectored(u_int vector,int dest)327 xen_pv_lapic_ipi_vectored(u_int vector, int dest)
328 {
329 xen_intr_handle_t *ipi_handle;
330 int ipi_idx, to_cpu, self;
331 static bool pvnmi = true;
332
333 if (vector >= IPI_NMI_FIRST) {
334 if (pvnmi) {
335 int rc = send_nmi(dest);
336
337 if (rc != 0) {
338 printf(
339 "Sending NMI using hypercall failed (%d) switching to APIC\n", rc);
340 pvnmi = false;
341 native_ipi_vectored(vector, dest);
342 }
343 } else
344 native_ipi_vectored(vector, dest);
345
346 return;
347 }
348
349 ipi_idx = IPI_TO_IDX(vector);
350 if (ipi_idx >= nitems(xen_ipis))
351 panic("IPI out of range");
352
353 switch(dest) {
354 case APIC_IPI_DEST_SELF:
355 ipi_handle = DPCPU_GET(ipi_handle);
356 xen_intr_signal(ipi_handle[ipi_idx]);
357 break;
358 case APIC_IPI_DEST_ALL:
359 CPU_FOREACH(to_cpu) {
360 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
361 xen_intr_signal(ipi_handle[ipi_idx]);
362 }
363 break;
364 case APIC_IPI_DEST_OTHERS:
365 self = PCPU_GET(cpuid);
366 CPU_FOREACH(to_cpu) {
367 if (to_cpu != self) {
368 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
369 xen_intr_signal(ipi_handle[ipi_idx]);
370 }
371 }
372 break;
373 default:
374 to_cpu = apic_cpuid(dest);
375 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
376 xen_intr_signal(ipi_handle[ipi_idx]);
377 break;
378 }
379 }
380
381 static int
xen_pv_lapic_ipi_wait(int delay)382 xen_pv_lapic_ipi_wait(int delay)
383 {
384
385 XEN_APIC_UNSUPPORTED;
386 return (0);
387 }
388 #endif /* SMP */
389
390 static int
xen_pv_lapic_ipi_alloc(inthand_t * ipifunc)391 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc)
392 {
393
394 XEN_APIC_UNSUPPORTED;
395 return (-1);
396 }
397
398 static void
xen_pv_lapic_ipi_free(int vector)399 xen_pv_lapic_ipi_free(int vector)
400 {
401
402 XEN_APIC_UNSUPPORTED;
403 }
404
405 static int
xen_pv_lapic_set_lvt_mask(u_int apic_id,u_int lvt,u_char masked)406 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked)
407 {
408
409 XEN_APIC_UNSUPPORTED;
410 return (0);
411 }
412
413 static int
xen_pv_lapic_set_lvt_mode(u_int apic_id,u_int lvt,uint32_t mode)414 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode)
415 {
416
417 XEN_APIC_UNSUPPORTED;
418 return (0);
419 }
420
421 static int
xen_pv_lapic_set_lvt_polarity(u_int apic_id,u_int lvt,enum intr_polarity pol)422 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol)
423 {
424
425 XEN_APIC_UNSUPPORTED;
426 return (0);
427 }
428
429 static int
xen_pv_lapic_set_lvt_triggermode(u_int apic_id,u_int lvt,enum intr_trigger trigger)430 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
431 enum intr_trigger trigger)
432 {
433
434 XEN_APIC_UNSUPPORTED;
435 return (0);
436 }
437
438 /* Xen apic_ops implementation */
439 struct apic_ops xen_apic_ops = {
440 .create = xen_pv_lapic_create,
441 .init = xen_pv_lapic_init,
442 .xapic_mode = xen_pv_lapic_disable,
443 .is_x2apic = xen_pv_lapic_is_x2apic,
444 .setup = xen_pv_lapic_setup,
445 .dump = xen_pv_lapic_dump,
446 .disable = xen_pv_lapic_disable,
447 .eoi = xen_pv_lapic_eoi,
448 .id = xen_pv_lapic_id,
449 .intr_pending = xen_pv_lapic_intr_pending,
450 .set_logical_id = xen_pv_lapic_set_logical_id,
451 .cpuid = xen_pv_apic_cpuid,
452 .alloc_vector = xen_pv_apic_alloc_vector,
453 .alloc_vectors = xen_pv_apic_alloc_vectors,
454 .enable_vector = xen_pv_apic_enable_vector,
455 .disable_vector = xen_pv_apic_disable_vector,
456 .free_vector = xen_pv_apic_free_vector,
457 .calibrate_timer = xen_pv_lapic_calibrate_timer,
458 .enable_pmc = xen_pv_lapic_enable_pmc,
459 .disable_pmc = xen_pv_lapic_disable_pmc,
460 .reenable_pmc = xen_pv_lapic_reenable_pmc,
461 .enable_cmc = xen_pv_lapic_enable_cmc,
462 #ifdef SMP
463 .ipi_raw = xen_pv_lapic_ipi_raw,
464 .ipi_vectored = xen_pv_lapic_ipi_vectored,
465 .ipi_wait = xen_pv_lapic_ipi_wait,
466 #endif
467 .ipi_alloc = xen_pv_lapic_ipi_alloc,
468 .ipi_free = xen_pv_lapic_ipi_free,
469 .set_lvt_mask = xen_pv_lapic_set_lvt_mask,
470 .set_lvt_mode = xen_pv_lapic_set_lvt_mode,
471 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity,
472 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode,
473 };
474
475 #ifdef SMP
476 /*---------------------------- XEN PV IPI Handlers ---------------------------*/
477 /*
478 * These are C clones of the ASM functions found in apic_vector.
479 */
480 static int
xen_ipi_bitmap_handler(void * arg)481 xen_ipi_bitmap_handler(void *arg)
482 {
483 struct trapframe *frame;
484
485 frame = arg;
486 ipi_bitmap_handler(*frame);
487 return (FILTER_HANDLED);
488 }
489
490 static int
xen_smp_rendezvous_action(void * arg)491 xen_smp_rendezvous_action(void *arg)
492 {
493 #ifdef COUNT_IPIS
494 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
495 #endif /* COUNT_IPIS */
496
497 smp_rendezvous_action();
498 return (FILTER_HANDLED);
499 }
500
501 #ifdef __amd64__
502 static int
xen_invlop(void * arg)503 xen_invlop(void *arg)
504 {
505
506 invlop_handler();
507 return (FILTER_HANDLED);
508 }
509
510 #else /* __i386__ */
511
512 static int
xen_invltlb(void * arg)513 xen_invltlb(void *arg)
514 {
515
516 invltlb_handler();
517 return (FILTER_HANDLED);
518 }
519
520 static int
xen_invlpg(void * arg)521 xen_invlpg(void *arg)
522 {
523
524 invlpg_handler();
525 return (FILTER_HANDLED);
526 }
527
528 static int
xen_invlrng(void * arg)529 xen_invlrng(void *arg)
530 {
531
532 invlrng_handler();
533 return (FILTER_HANDLED);
534 }
535
536 static int
xen_invlcache(void * arg)537 xen_invlcache(void *arg)
538 {
539
540 invlcache_handler();
541 return (FILTER_HANDLED);
542 }
543 #endif /* __amd64__ */
544
545 static int
xen_cpustop_handler(void * arg)546 xen_cpustop_handler(void *arg)
547 {
548
549 cpustop_handler();
550 return (FILTER_HANDLED);
551 }
552
553 static int
xen_cpususpend_handler(void * arg)554 xen_cpususpend_handler(void *arg)
555 {
556
557 cpususpend_handler();
558 return (FILTER_HANDLED);
559 }
560
561 static int
xen_ipi_swi_handler(void * arg)562 xen_ipi_swi_handler(void *arg)
563 {
564 struct trapframe *frame = arg;
565
566 ipi_swi_handler(*frame);
567 return (FILTER_HANDLED);
568 }
569
570 /*----------------------------- XEN PV IPI setup -----------------------------*/
571 /*
572 * Those functions are provided outside of the Xen PV APIC implementation
573 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC,
574 * because on PVHVM there's an emulated LAPIC provided by Xen.
575 */
576 static void
xen_cpu_ipi_init(int cpu)577 xen_cpu_ipi_init(int cpu)
578 {
579 xen_intr_handle_t *ipi_handle;
580 const struct xen_ipi_handler *ipi;
581 int idx, rc;
582
583 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
584
585 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
586 if (ipi->filter == NULL) {
587 ipi_handle[idx] = NULL;
588 continue;
589 }
590
591 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter,
592 INTR_TYPE_TTY, &ipi_handle[idx]);
593 if (rc != 0)
594 panic("Unable to allocate a XEN IPI port");
595 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
596 }
597 }
598
599 static void
xen_setup_cpus(void)600 xen_setup_cpus(void)
601 {
602 int i;
603
604 if (!xen_vector_callback_enabled)
605 return;
606
607 CPU_FOREACH(i)
608 xen_cpu_ipi_init(i);
609
610 /* Set the xen pv ipi ops to replace the native ones */
611 native_ipi_vectored = apic_ops.ipi_vectored;
612 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored;
613 }
614
615 /* Switch to using PV IPIs as soon as the vcpu_id is set. */
616 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL);
617 #endif /* SMP */
618