1 /*-
2 * Copyright (c) 1996, by Steve Passe
3 * Copyright (c) 2008, by Kip Macy
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. The name of the developer may NOT be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD: stable/10/sys/i386/xen/mp_machdep.c 278423 2015-02-08 23:04:32Z marius $");
29
30 #include "opt_apic.h"
31 #include "opt_cpu.h"
32 #include "opt_kstack_pages.h"
33 #include "opt_mp_watchdog.h"
34 #include "opt_pmap.h"
35 #include "opt_sched.h"
36 #include "opt_smp.h"
37
38 #if !defined(lint)
39 #if !defined(SMP)
40 #error How did you get here?
41 #endif
42
43 #ifndef DEV_APIC
44 #error The apic device is required for SMP, add "device apic" to your config file.
45 #endif
46 #if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
47 #error SMP not supported with CPU_DISABLE_CMPXCHG
48 #endif
49 #endif /* not lint */
50
51 #include <sys/param.h>
52 #include <sys/systm.h>
53 #include <sys/bus.h>
54 #include <sys/cons.h> /* cngetc() */
55 #include <sys/cpuset.h>
56 #ifdef GPROF
57 #include <sys/gmon.h>
58 #endif
59 #include <sys/kernel.h>
60 #include <sys/ktr.h>
61 #include <sys/lock.h>
62 #include <sys/malloc.h>
63 #include <sys/memrange.h>
64 #include <sys/mutex.h>
65 #include <sys/pcpu.h>
66 #include <sys/proc.h>
67 #include <sys/rwlock.h>
68 #include <sys/sched.h>
69 #include <sys/smp.h>
70 #include <sys/sysctl.h>
71
72 #include <vm/vm.h>
73 #include <vm/vm_param.h>
74 #include <vm/pmap.h>
75 #include <vm/vm_kern.h>
76 #include <vm/vm_extern.h>
77 #include <vm/vm_page.h>
78
79 #include <x86/apicreg.h>
80 #include <machine/md_var.h>
81 #include <machine/mp_watchdog.h>
82 #include <machine/pcb.h>
83 #include <machine/psl.h>
84 #include <machine/smp.h>
85 #include <machine/specialreg.h>
86 #include <machine/pcpu.h>
87
88 #include <xen/xen-os.h>
89 #include <xen/evtchn.h>
90 #include <xen/xen_intr.h>
91 #include <xen/hypervisor.h>
92 #include <xen/interface/vcpu.h>
93
94 /*---------------------------- Extern Declarations ---------------------------*/
95 extern struct pcpu __pcpu[];
96
97 extern void Xhypervisor_callback(void);
98 extern void failsafe_callback(void);
99 extern void pmap_lazyfix_action(void);
100
101 /*--------------------------- Forward Declarations ---------------------------*/
102 static driver_filter_t smp_reschedule_interrupt;
103 static driver_filter_t smp_call_function_interrupt;
104 static void assign_cpu_ids(void);
105 static void set_interrupt_apic_ids(void);
106 static int start_all_aps(void);
107 static int start_ap(int apic_id);
108 static void release_aps(void *dummy);
109
110 /*---------------------------------- Macros ----------------------------------*/
111 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
112
113 /*-------------------------------- Local Types -------------------------------*/
114 typedef void call_data_func_t(uintptr_t , uintptr_t);
115
116 struct cpu_info {
117 int cpu_present:1;
118 int cpu_bsp:1;
119 int cpu_disabled:1;
120 };
121
122 struct xen_ipi_handler
123 {
124 driver_filter_t *filter;
125 const char *description;
126 };
127
128 enum {
129 RESCHEDULE_VECTOR,
130 CALL_FUNCTION_VECTOR,
131 };
132
133 /*-------------------------------- Global Data -------------------------------*/
134 static u_int hyperthreading_cpus;
135 static cpuset_t hyperthreading_cpus_mask;
136
137 int mp_naps; /* # of Applications processors */
138 int boot_cpu_id = -1; /* designated BSP */
139
140 static int bootAP;
141 static union descriptor *bootAPgdt;
142
143 /* Free these after use */
144 void *bootstacks[MAXCPU];
145
146 struct pcb stoppcbs[MAXCPU];
147
148 /* Variables needed for SMP tlb shootdown. */
149 vm_offset_t smp_tlb_addr1;
150 vm_offset_t smp_tlb_addr2;
151 volatile int smp_tlb_wait;
152
153 static u_int logical_cpus;
154 static volatile cpuset_t ipi_nmi_pending;
155
156 /* used to hold the AP's until we are ready to release them */
157 static struct mtx ap_boot_mtx;
158
159 /* Set to 1 once we're ready to let the APs out of the pen. */
160 static volatile int aps_ready = 0;
161
162 /*
163 * Store data from cpu_add() until later in the boot when we actually setup
164 * the APs.
165 */
166 static struct cpu_info cpu_info[MAX_APIC_ID + 1];
167 int cpu_apic_ids[MAXCPU];
168 int apic_cpuids[MAX_APIC_ID + 1];
169
170 /* Holds pending bitmap based IPIs per CPU */
171 static volatile u_int cpu_ipi_pending[MAXCPU];
172
173 static int cpu_logical;
174 static int cpu_cores;
175
176 static const struct xen_ipi_handler xen_ipis[] =
177 {
178 [RESCHEDULE_VECTOR] = { smp_reschedule_interrupt, "resched" },
179 [CALL_FUNCTION_VECTOR] = { smp_call_function_interrupt,"callfunc" }
180 };
181
182 /*------------------------------- Per-CPU Data -------------------------------*/
183 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
184 DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
185
186 /*------------------------------ Implementation ------------------------------*/
187 struct cpu_group *
cpu_topo(void)188 cpu_topo(void)
189 {
190 if (cpu_cores == 0)
191 cpu_cores = 1;
192 if (cpu_logical == 0)
193 cpu_logical = 1;
194 if (mp_ncpus % (cpu_cores * cpu_logical) != 0) {
195 printf("WARNING: Non-uniform processors.\n");
196 printf("WARNING: Using suboptimal topology.\n");
197 return (smp_topo_none());
198 }
199 /*
200 * No multi-core or hyper-threaded.
201 */
202 if (cpu_logical * cpu_cores == 1)
203 return (smp_topo_none());
204 /*
205 * Only HTT no multi-core.
206 */
207 if (cpu_logical > 1 && cpu_cores == 1)
208 return (smp_topo_1level(CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
209 /*
210 * Only multi-core no HTT.
211 */
212 if (cpu_cores > 1 && cpu_logical == 1)
213 return (smp_topo_1level(CG_SHARE_NONE, cpu_cores, 0));
214 /*
215 * Both HTT and multi-core.
216 */
217 return (smp_topo_2level(CG_SHARE_NONE, cpu_cores,
218 CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
219 }
220
221 /*
222 * Calculate usable address in base memory for AP trampoline code.
223 */
224 u_int
mp_bootaddress(u_int basemem)225 mp_bootaddress(u_int basemem)
226 {
227
228 return (basemem);
229 }
230
231 void
cpu_add(u_int apic_id,char boot_cpu)232 cpu_add(u_int apic_id, char boot_cpu)
233 {
234
235 if (apic_id > MAX_APIC_ID) {
236 panic("SMP: APIC ID %d too high", apic_id);
237 return;
238 }
239 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice",
240 apic_id));
241 cpu_info[apic_id].cpu_present = 1;
242 if (boot_cpu) {
243 KASSERT(boot_cpu_id == -1,
244 ("CPU %d claims to be BSP, but CPU %d already is", apic_id,
245 boot_cpu_id));
246 boot_cpu_id = apic_id;
247 cpu_info[apic_id].cpu_bsp = 1;
248 }
249 if (mp_ncpus < MAXCPU)
250 mp_ncpus++;
251 if (bootverbose)
252 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" :
253 "AP");
254 }
255
256 void
cpu_mp_setmaxid(void)257 cpu_mp_setmaxid(void)
258 {
259
260 mp_maxid = MAXCPU - 1;
261 }
262
263 int
cpu_mp_probe(void)264 cpu_mp_probe(void)
265 {
266
267 /*
268 * Always record BSP in CPU map so that the mbuf init code works
269 * correctly.
270 */
271 CPU_SETOF(0, &all_cpus);
272 if (mp_ncpus == 0) {
273 /*
274 * No CPUs were found, so this must be a UP system. Setup
275 * the variables to represent a system with a single CPU
276 * with an id of 0.
277 */
278 mp_ncpus = 1;
279 return (0);
280 }
281
282 /* At least one CPU was found. */
283 if (mp_ncpus == 1) {
284 /*
285 * One CPU was found, so this must be a UP system with
286 * an I/O APIC.
287 */
288 return (0);
289 }
290
291 /* At least two CPUs were found. */
292 return (1);
293 }
294
295 /*
296 * Initialize the IPI handlers and start up the AP's.
297 */
298 void
cpu_mp_start(void)299 cpu_mp_start(void)
300 {
301 int i;
302
303 /* Initialize the logical ID to APIC ID table. */
304 for (i = 0; i < MAXCPU; i++) {
305 cpu_apic_ids[i] = -1;
306 cpu_ipi_pending[i] = 0;
307 }
308
309 /* Set boot_cpu_id if needed. */
310 if (boot_cpu_id == -1) {
311 boot_cpu_id = PCPU_GET(apic_id);
312 cpu_info[boot_cpu_id].cpu_bsp = 1;
313 } else
314 KASSERT(boot_cpu_id == PCPU_GET(apic_id),
315 ("BSP's APIC ID doesn't match boot_cpu_id"));
316 cpu_apic_ids[0] = boot_cpu_id;
317 apic_cpuids[boot_cpu_id] = 0;
318
319 assign_cpu_ids();
320
321 /* Start each Application Processor */
322 start_all_aps();
323
324 /* Setup the initial logical CPUs info. */
325 logical_cpus = 0;
326 CPU_ZERO(&logical_cpus_mask);
327 if (cpu_feature & CPUID_HTT)
328 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
329
330 set_interrupt_apic_ids();
331 }
332
333
334 static void
iv_rendezvous(uintptr_t a,uintptr_t b)335 iv_rendezvous(uintptr_t a, uintptr_t b)
336 {
337 smp_rendezvous_action();
338 }
339
340 static void
iv_invltlb(uintptr_t a,uintptr_t b)341 iv_invltlb(uintptr_t a, uintptr_t b)
342 {
343 xen_tlb_flush();
344 }
345
346 static void
iv_invlpg(uintptr_t a,uintptr_t b)347 iv_invlpg(uintptr_t a, uintptr_t b)
348 {
349 xen_invlpg(a);
350 }
351
352 static void
iv_invlrng(uintptr_t a,uintptr_t b)353 iv_invlrng(uintptr_t a, uintptr_t b)
354 {
355 vm_offset_t start = (vm_offset_t)a;
356 vm_offset_t end = (vm_offset_t)b;
357
358 while (start < end) {
359 xen_invlpg(start);
360 start += PAGE_SIZE;
361 }
362 }
363
364
365 static void
iv_invlcache(uintptr_t a,uintptr_t b)366 iv_invlcache(uintptr_t a, uintptr_t b)
367 {
368
369 wbinvd();
370 atomic_add_int(&smp_tlb_wait, 1);
371 }
372
373 static void
iv_lazypmap(uintptr_t a,uintptr_t b)374 iv_lazypmap(uintptr_t a, uintptr_t b)
375 {
376 pmap_lazyfix_action();
377 atomic_add_int(&smp_tlb_wait, 1);
378 }
379
380 /*
381 * These start from "IPI offset" APIC_IPI_INTS
382 */
383 static call_data_func_t *ipi_vectors[6] =
384 {
385 iv_rendezvous,
386 iv_invltlb,
387 iv_invlpg,
388 iv_invlrng,
389 iv_invlcache,
390 iv_lazypmap,
391 };
392
393 /*
394 * Reschedule call back. Nothing to do,
395 * all the work is done automatically when
396 * we return from the interrupt.
397 */
398 static int
smp_reschedule_interrupt(void * unused)399 smp_reschedule_interrupt(void *unused)
400 {
401 int cpu = PCPU_GET(cpuid);
402 u_int ipi_bitmap;
403
404 ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
405
406 if (ipi_bitmap & (1 << IPI_PREEMPT)) {
407 #ifdef COUNT_IPIS
408 (*ipi_preempt_counts[cpu])++;
409 #endif
410 sched_preempt(curthread);
411 }
412
413 if (ipi_bitmap & (1 << IPI_AST)) {
414 #ifdef COUNT_IPIS
415 (*ipi_ast_counts[cpu])++;
416 #endif
417 /* Nothing to do for AST */
418 }
419 return (FILTER_HANDLED);
420 }
421
422 struct _call_data {
423 uint16_t func_id;
424 uint16_t wait;
425 uintptr_t arg1;
426 uintptr_t arg2;
427 atomic_t started;
428 atomic_t finished;
429 };
430
431 static struct _call_data *call_data;
432
433 static int
smp_call_function_interrupt(void * unused)434 smp_call_function_interrupt(void *unused)
435 {
436 call_data_func_t *func;
437 uintptr_t arg1 = call_data->arg1;
438 uintptr_t arg2 = call_data->arg2;
439 int wait = call_data->wait;
440 atomic_t *started = &call_data->started;
441 atomic_t *finished = &call_data->finished;
442
443 /* We only handle function IPIs, not bitmap IPIs */
444 if (call_data->func_id < APIC_IPI_INTS ||
445 call_data->func_id > IPI_BITMAP_VECTOR)
446 panic("invalid function id %u", call_data->func_id);
447
448 func = ipi_vectors[IPI_TO_IDX(call_data->func_id)];
449 /*
450 * Notify initiating CPU that I've grabbed the data and am
451 * about to execute the function
452 */
453 mb();
454 atomic_inc(started);
455 /*
456 * At this point the info structure may be out of scope unless wait==1
457 */
458 (*func)(arg1, arg2);
459
460 if (wait) {
461 mb();
462 atomic_inc(finished);
463 }
464 atomic_add_int(&smp_tlb_wait, 1);
465 return (FILTER_HANDLED);
466 }
467
468 /*
469 * Print various information about the SMP system hardware and setup.
470 */
471 void
cpu_mp_announce(void)472 cpu_mp_announce(void)
473 {
474 int i, x;
475
476 /* List CPUs */
477 printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
478 for (i = 1, x = 0; x <= MAX_APIC_ID; x++) {
479 if (!cpu_info[x].cpu_present || cpu_info[x].cpu_bsp)
480 continue;
481 if (cpu_info[x].cpu_disabled)
482 printf(" cpu (AP): APIC ID: %2d (disabled)\n", x);
483 else {
484 KASSERT(i < mp_ncpus,
485 ("mp_ncpus and actual cpus are out of whack"));
486 printf(" cpu%d (AP): APIC ID: %2d\n", i++, x);
487 }
488 }
489 }
490
491 static int
xen_smp_cpu_init(unsigned int cpu)492 xen_smp_cpu_init(unsigned int cpu)
493 {
494 xen_intr_handle_t *ipi_handle;
495 const struct xen_ipi_handler *ipi;
496 int idx, rc;
497
498 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
499 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
500
501 /*
502 * The PCPU variable pc_device is not initialized on i386 PV,
503 * so we have to use the root_bus device in order to setup
504 * the IPIs.
505 */
506 rc = xen_intr_alloc_and_bind_ipi(root_bus, cpu,
507 ipi->filter, INTR_TYPE_TTY, &ipi_handle[idx]);
508 if (rc != 0) {
509 printf("Unable to allocate a XEN IPI port. "
510 "Error %d\n", rc);
511 break;
512 }
513 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
514 }
515
516 for (;idx < nitems(xen_ipis); idx++)
517 ipi_handle[idx] = NULL;
518
519 if (rc == 0)
520 return (0);
521
522 /* Either all are successfully mapped, or none at all. */
523 for (idx = 0; idx < nitems(xen_ipis); idx++) {
524 if (ipi_handle[idx] == NULL)
525 continue;
526
527 xen_intr_unbind(ipi_handle[idx]);
528 ipi_handle[idx] = NULL;
529 }
530
531 return (rc);
532 }
533
534 static void
xen_smp_intr_init_cpus(void * unused)535 xen_smp_intr_init_cpus(void *unused)
536 {
537 int i;
538
539 for (i = 0; i < mp_ncpus; i++)
540 xen_smp_cpu_init(i);
541 }
542
543 static void
xen_smp_intr_setup_cpus(void * unused)544 xen_smp_intr_setup_cpus(void *unused)
545 {
546 int i;
547
548 for (i = 0; i < mp_ncpus; i++)
549 DPCPU_ID_SET(i, vcpu_info,
550 &HYPERVISOR_shared_info->vcpu_info[i]);
551 }
552
553 #define MTOPSIZE (1<<(14 + PAGE_SHIFT))
554
555 /*
556 * AP CPU's call this to initialize themselves.
557 */
558 void
init_secondary(void)559 init_secondary(void)
560 {
561 vm_offset_t addr;
562 u_int cpuid;
563 int gsel_tss;
564
565
566 /* bootAP is set in start_ap() to our ID. */
567 PCPU_SET(currentldt, _default_ldt);
568 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
569 #if 0
570 gdt[bootAP * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
571 #endif
572 PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
573 PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
574 PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
575 #if 0
576 PCPU_SET(tss_gdt, &gdt[bootAP * NGDT + GPROC0_SEL].sd);
577
578 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt));
579 #endif
580 PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd);
581
582 /*
583 * Set to a known state:
584 * Set by mpboot.s: CR0_PG, CR0_PE
585 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
586 */
587 /*
588 * signal our startup to the BSP.
589 */
590 mp_naps++;
591
592 /* Spin until the BSP releases the AP's. */
593 while (!aps_ready)
594 ia32_pause();
595
596 /* BSP may have changed PTD while we were waiting */
597 invltlb();
598 for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE)
599 invlpg(addr);
600
601 #if 0
602 /* set up SSE/NX */
603 initializecpu();
604 #endif
605
606 /* set up FPU state on the AP */
607 npxinit(false);
608 #if 0
609 /* A quick check from sanity claus */
610 if (PCPU_GET(apic_id) != lapic_id()) {
611 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
612 printf("SMP: actual apic_id = %d\n", lapic_id());
613 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
614 panic("cpuid mismatch! boom!!");
615 }
616 #endif
617
618 /* Initialize curthread. */
619 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
620 PCPU_SET(curthread, PCPU_GET(idlethread));
621
622 mtx_lock_spin(&ap_boot_mtx);
623 #if 0
624
625 /* Init local apic for irq's */
626 lapic_setup(1);
627 #endif
628 smp_cpus++;
629
630 cpuid = PCPU_GET(cpuid);
631 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
632 printf("SMP: AP CPU #%d Launched!\n", cpuid);
633
634 /* Determine if we are a logical CPU. */
635 if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0)
636 CPU_SET(cpuid, &logical_cpus_mask);
637
638 /* Determine if we are a hyperthread. */
639 if (hyperthreading_cpus > 1 &&
640 PCPU_GET(apic_id) % hyperthreading_cpus != 0)
641 CPU_SET(cpuid, &hyperthreading_cpus_mask);
642 #if 0
643 if (bootverbose)
644 lapic_dump("AP");
645 #endif
646 if (smp_cpus == mp_ncpus) {
647 /* enable IPI's, tlb shootdown, freezes etc */
648 atomic_store_rel_int(&smp_started, 1);
649 }
650
651 mtx_unlock_spin(&ap_boot_mtx);
652
653 /* wait until all the AP's are up */
654 while (smp_started == 0)
655 ia32_pause();
656
657 PCPU_SET(curthread, PCPU_GET(idlethread));
658
659 /* Start per-CPU event timers. */
660 cpu_initclocks_ap();
661
662 /* enter the scheduler */
663 sched_throw(NULL);
664
665 panic("scheduler returned us to %s", __func__);
666 /* NOTREACHED */
667 }
668
669 /*******************************************************************
670 * local functions and data
671 */
672
673 /*
674 * We tell the I/O APIC code about all the CPUs we want to receive
675 * interrupts. If we don't want certain CPUs to receive IRQs we
676 * can simply not tell the I/O APIC code about them in this function.
677 * We also do not tell it about the BSP since it tells itself about
678 * the BSP internally to work with UP kernels and on UP machines.
679 */
680 static void
set_interrupt_apic_ids(void)681 set_interrupt_apic_ids(void)
682 {
683 u_int i, apic_id;
684
685 for (i = 0; i < MAXCPU; i++) {
686 apic_id = cpu_apic_ids[i];
687 if (apic_id == -1)
688 continue;
689 if (cpu_info[apic_id].cpu_bsp)
690 continue;
691 if (cpu_info[apic_id].cpu_disabled)
692 continue;
693
694 /* Don't let hyperthreads service interrupts. */
695 if (hyperthreading_cpus > 1 &&
696 apic_id % hyperthreading_cpus != 0)
697 continue;
698
699 intr_add_cpu(i);
700 }
701 }
702
703 /*
704 * Assign logical CPU IDs to local APICs.
705 */
706 static void
assign_cpu_ids(void)707 assign_cpu_ids(void)
708 {
709 u_int i;
710
711 /* Check for explicitly disabled CPUs. */
712 for (i = 0; i <= MAX_APIC_ID; i++) {
713 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp)
714 continue;
715
716 /* Don't use this CPU if it has been disabled by a tunable. */
717 if (resource_disabled("lapic", i)) {
718 cpu_info[i].cpu_disabled = 1;
719 continue;
720 }
721 }
722
723 /*
724 * Assign CPU IDs to local APIC IDs and disable any CPUs
725 * beyond MAXCPU. CPU 0 has already been assigned to the BSP,
726 * so we only have to assign IDs for APs.
727 */
728 mp_ncpus = 1;
729 for (i = 0; i <= MAX_APIC_ID; i++) {
730 if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp ||
731 cpu_info[i].cpu_disabled)
732 continue;
733
734 if (mp_ncpus < MAXCPU) {
735 cpu_apic_ids[mp_ncpus] = i;
736 apic_cpuids[i] = mp_ncpus;
737 mp_ncpus++;
738 } else
739 cpu_info[i].cpu_disabled = 1;
740 }
741 KASSERT(mp_maxid >= mp_ncpus - 1,
742 ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid,
743 mp_ncpus));
744 }
745
746 /*
747 * start each AP in our list
748 */
749 /* Lowest 1MB is already mapped: don't touch*/
750 #define TMPMAP_START 1
751 int
start_all_aps(void)752 start_all_aps(void)
753 {
754 int x,apic_id, cpu;
755 struct pcpu *pc;
756
757 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
758
759 /* set up temporary P==V mapping for AP boot */
760 /* XXX this is a hack, we should boot the AP on its own stack/PTD */
761
762 /* start each AP */
763 for (cpu = 1; cpu < mp_ncpus; cpu++) {
764 apic_id = cpu_apic_ids[cpu];
765
766
767 bootAP = cpu;
768 bootAPgdt = gdt + (512*cpu);
769
770 /* Get per-cpu data */
771 pc = &__pcpu[bootAP];
772 pcpu_init(pc, bootAP, sizeof(struct pcpu));
773 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
774 M_WAITOK | M_ZERO), bootAP);
775 pc->pc_apic_id = cpu_apic_ids[bootAP];
776 pc->pc_vcpu_id = cpu_apic_ids[bootAP];
777 pc->pc_prvspace = pc;
778 pc->pc_curthread = 0;
779
780 gdt_segs[GPRIV_SEL].ssd_base = (int) pc;
781 gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss;
782
783 PT_SET_MA(bootAPgdt, VTOM(bootAPgdt) | PG_V | PG_RW);
784 bzero(bootAPgdt, PAGE_SIZE);
785 for (x = 0; x < NGDT; x++)
786 ssdtosd(&gdt_segs[x], &bootAPgdt[x].sd);
787 PT_SET_MA(bootAPgdt, vtomach(bootAPgdt) | PG_V);
788 #ifdef notyet
789
790 if (HYPERVISOR_vcpu_op(VCPUOP_get_physid, cpu, &cpu_id) == 0) {
791 apicid = xen_vcpu_physid_to_x86_apicid(cpu_id.phys_id);
792 acpiid = xen_vcpu_physid_to_x86_acpiid(cpu_id.phys_id);
793 #ifdef CONFIG_ACPI
794 if (acpiid != 0xff)
795 x86_acpiid_to_apicid[acpiid] = apicid;
796 #endif
797 }
798 #endif
799
800 /* attempt to start the Application Processor */
801 if (!start_ap(cpu)) {
802 printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
803 /* better panic as the AP may be running loose */
804 printf("panic y/n? [y] ");
805 if (cngetc() != 'n')
806 panic("bye-bye");
807 }
808
809 CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
810 }
811
812
813 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
814
815 /* number of APs actually started */
816 return (mp_naps);
817 }
818
819 extern uint8_t *pcpu_boot_stack;
820 extern trap_info_t trap_table[];
821
822 static void
smp_trap_init(trap_info_t * trap_ctxt)823 smp_trap_init(trap_info_t *trap_ctxt)
824 {
825 const trap_info_t *t = trap_table;
826
827 for (t = trap_table; t->address; t++) {
828 trap_ctxt[t->vector].flags = t->flags;
829 trap_ctxt[t->vector].cs = t->cs;
830 trap_ctxt[t->vector].address = t->address;
831 }
832 }
833
834 extern struct rwlock pvh_global_lock;
835 extern int nkpt;
836 static void
cpu_initialize_context(unsigned int cpu)837 cpu_initialize_context(unsigned int cpu)
838 {
839 /* vcpu_guest_context_t is too large to allocate on the stack.
840 * Hence we allocate statically and protect it with a lock */
841 vm_page_t m[NPGPTD + 2];
842 static vcpu_guest_context_t ctxt;
843 vm_offset_t boot_stack;
844 vm_offset_t newPTD;
845 vm_paddr_t ma[NPGPTD];
846 int i;
847
848 /*
849 * Page 0,[0-3] PTD
850 * Page 1, [4] boot stack
851 * Page [5] PDPT
852 *
853 */
854 for (i = 0; i < NPGPTD + 2; i++) {
855 m[i] = vm_page_alloc(NULL, 0,
856 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
857 VM_ALLOC_ZERO);
858
859 pmap_zero_page(m[i]);
860
861 }
862 boot_stack = kva_alloc(PAGE_SIZE);
863 newPTD = kva_alloc(NPGPTD * PAGE_SIZE);
864 ma[0] = VM_PAGE_TO_MACH(m[0])|PG_V;
865
866 #ifdef PAE
867 pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD + 1]));
868 for (i = 0; i < NPGPTD; i++) {
869 ((vm_paddr_t *)boot_stack)[i] =
870 ma[i] = VM_PAGE_TO_MACH(m[i])|PG_V;
871 }
872 #endif
873
874 /*
875 * Copy cpu0 IdlePTD to new IdlePTD - copying only
876 * kernel mappings
877 */
878 pmap_qenter(newPTD, m, 4);
879
880 memcpy((uint8_t *)newPTD + KPTDI*sizeof(vm_paddr_t),
881 (uint8_t *)PTOV(IdlePTD) + KPTDI*sizeof(vm_paddr_t),
882 nkpt*sizeof(vm_paddr_t));
883
884 pmap_qremove(newPTD, 4);
885 kva_free(newPTD, 4 * PAGE_SIZE);
886 /*
887 * map actual idle stack to boot_stack
888 */
889 pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD]));
890
891
892 xen_pgdpt_pin(VM_PAGE_TO_MACH(m[NPGPTD + 1]));
893 rw_wlock(&pvh_global_lock);
894 for (i = 0; i < 4; i++) {
895 int pdir = (PTDPTDI + i) / NPDEPG;
896 int curoffset = (PTDPTDI + i) % NPDEPG;
897
898 xen_queue_pt_update((vm_paddr_t)
899 ((ma[pdir] & ~PG_V) + (curoffset*sizeof(vm_paddr_t))),
900 ma[i]);
901 }
902 PT_UPDATES_FLUSH();
903 rw_wunlock(&pvh_global_lock);
904
905 memset(&ctxt, 0, sizeof(ctxt));
906 ctxt.flags = VGCF_IN_KERNEL;
907 ctxt.user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
908 ctxt.user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
909 ctxt.user_regs.fs = GSEL(GPRIV_SEL, SEL_KPL);
910 ctxt.user_regs.gs = GSEL(GDATA_SEL, SEL_KPL);
911 ctxt.user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
912 ctxt.user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
913 ctxt.user_regs.eip = (unsigned long)init_secondary;
914 ctxt.user_regs.eflags = PSL_KERNEL | 0x1000; /* IOPL_RING1 */
915
916 memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
917
918 smp_trap_init(ctxt.trap_ctxt);
919
920 ctxt.ldt_ents = 0;
921 ctxt.gdt_frames[0] =
922 (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
923 ctxt.gdt_ents = 512;
924
925 #ifdef __i386__
926 ctxt.user_regs.esp = boot_stack + PAGE_SIZE;
927
928 ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
929 ctxt.kernel_sp = boot_stack + PAGE_SIZE;
930
931 ctxt.event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
932 ctxt.event_callback_eip = (unsigned long)Xhypervisor_callback;
933 ctxt.failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
934 ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
935
936 ctxt.ctrlreg[3] = VM_PAGE_TO_MACH(m[NPGPTD + 1]);
937 #else /* __x86_64__ */
938 ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
939 ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
940 ctxt.kernel_sp = idle->thread.rsp0;
941
942 ctxt.event_callback_eip = (unsigned long)hypervisor_callback;
943 ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
944 ctxt.syscall_callback_eip = (unsigned long)system_call;
945
946 ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
947
948 ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
949 #endif
950
951 printf("gdtpfn=%lx pdptpfn=%lx\n",
952 ctxt.gdt_frames[0],
953 ctxt.ctrlreg[3] >> PAGE_SHIFT);
954
955 PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, &ctxt));
956 DELAY(3000);
957 PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL));
958 }
959
960 /*
961 * This function starts the AP (application processor) identified
962 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
963 * to accomplish this. This is necessary because of the nuances
964 * of the different hardware we might encounter. It isn't pretty,
965 * but it seems to work.
966 */
967
968 int cpus;
969 static int
start_ap(int apic_id)970 start_ap(int apic_id)
971 {
972 int ms;
973
974 /* used as a watchpoint to signal AP startup */
975 cpus = mp_naps;
976
977 cpu_initialize_context(apic_id);
978
979 /* Wait up to 5 seconds for it to start. */
980 for (ms = 0; ms < 5000; ms++) {
981 if (mp_naps > cpus)
982 return (1); /* return SUCCESS */
983 DELAY(1000);
984 }
985 return (0); /* return FAILURE */
986 }
987
988 static void
ipi_pcpu(int cpu,u_int ipi)989 ipi_pcpu(int cpu, u_int ipi)
990 {
991 KASSERT((ipi <= nitems(xen_ipis)), ("invalid IPI"));
992 xen_intr_signal(DPCPU_ID_GET(cpu, ipi_handle[ipi]));
993 }
994
995 /*
996 * send an IPI to a specific CPU.
997 */
998 static void
ipi_send_cpu(int cpu,u_int ipi)999 ipi_send_cpu(int cpu, u_int ipi)
1000 {
1001 u_int bitmap, old_pending, new_pending;
1002
1003 if (IPI_IS_BITMAPED(ipi)) {
1004 bitmap = 1 << ipi;
1005 ipi = IPI_BITMAP_VECTOR;
1006 do {
1007 old_pending = cpu_ipi_pending[cpu];
1008 new_pending = old_pending | bitmap;
1009 } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
1010 old_pending, new_pending));
1011 if (!old_pending)
1012 ipi_pcpu(cpu, RESCHEDULE_VECTOR);
1013 } else {
1014 KASSERT(call_data != NULL, ("call_data not set"));
1015 ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
1016 }
1017 }
1018
1019 /*
1020 * Flush the TLB on all other CPU's
1021 */
1022 static void
smp_tlb_shootdown(u_int vector,vm_offset_t addr1,vm_offset_t addr2)1023 smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
1024 {
1025 u_int ncpu;
1026 struct _call_data data;
1027
1028 ncpu = mp_ncpus - 1; /* does not shootdown self */
1029 if (ncpu < 1)
1030 return; /* no other cpus */
1031 if (!(read_eflags() & PSL_I))
1032 panic("%s: interrupts disabled", __func__);
1033 mtx_lock_spin(&smp_ipi_mtx);
1034 KASSERT(call_data == NULL, ("call_data isn't null?!"));
1035 call_data = &data;
1036 call_data->func_id = vector;
1037 call_data->arg1 = addr1;
1038 call_data->arg2 = addr2;
1039 atomic_store_rel_int(&smp_tlb_wait, 0);
1040 ipi_all_but_self(vector);
1041 while (smp_tlb_wait < ncpu)
1042 ia32_pause();
1043 call_data = NULL;
1044 mtx_unlock_spin(&smp_ipi_mtx);
1045 }
1046
1047 static void
smp_targeted_tlb_shootdown(cpuset_t mask,u_int vector,vm_offset_t addr1,vm_offset_t addr2)1048 smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1,
1049 vm_offset_t addr2)
1050 {
1051 int cpu, ncpu, othercpus;
1052 struct _call_data data;
1053
1054 othercpus = mp_ncpus - 1;
1055 if (CPU_ISFULLSET(&mask)) {
1056 if (othercpus < 1)
1057 return;
1058 } else {
1059 CPU_CLR(PCPU_GET(cpuid), &mask);
1060 if (CPU_EMPTY(&mask))
1061 return;
1062 }
1063 if (!(read_eflags() & PSL_I))
1064 panic("%s: interrupts disabled", __func__);
1065 mtx_lock_spin(&smp_ipi_mtx);
1066 KASSERT(call_data == NULL, ("call_data isn't null?!"));
1067 call_data = &data;
1068 call_data->func_id = vector;
1069 call_data->arg1 = addr1;
1070 call_data->arg2 = addr2;
1071 atomic_store_rel_int(&smp_tlb_wait, 0);
1072 if (CPU_ISFULLSET(&mask)) {
1073 ncpu = othercpus;
1074 ipi_all_but_self(vector);
1075 } else {
1076 ncpu = 0;
1077 while ((cpu = CPU_FFS(&mask)) != 0) {
1078 cpu--;
1079 CPU_CLR(cpu, &mask);
1080 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu,
1081 vector);
1082 ipi_send_cpu(cpu, vector);
1083 ncpu++;
1084 }
1085 }
1086 while (smp_tlb_wait < ncpu)
1087 ia32_pause();
1088 call_data = NULL;
1089 mtx_unlock_spin(&smp_ipi_mtx);
1090 }
1091
1092 void
smp_cache_flush(void)1093 smp_cache_flush(void)
1094 {
1095
1096 if (smp_started)
1097 smp_tlb_shootdown(IPI_INVLCACHE, 0, 0);
1098 }
1099
1100 void
smp_invltlb(void)1101 smp_invltlb(void)
1102 {
1103
1104 if (smp_started) {
1105 smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
1106 }
1107 }
1108
1109 void
smp_invlpg(vm_offset_t addr)1110 smp_invlpg(vm_offset_t addr)
1111 {
1112
1113 if (smp_started) {
1114 smp_tlb_shootdown(IPI_INVLPG, addr, 0);
1115 }
1116 }
1117
1118 void
smp_invlpg_range(vm_offset_t addr1,vm_offset_t addr2)1119 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
1120 {
1121
1122 if (smp_started) {
1123 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
1124 }
1125 }
1126
1127 void
smp_masked_invltlb(cpuset_t mask)1128 smp_masked_invltlb(cpuset_t mask)
1129 {
1130
1131 if (smp_started) {
1132 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
1133 }
1134 }
1135
1136 void
smp_masked_invlpg(cpuset_t mask,vm_offset_t addr)1137 smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
1138 {
1139
1140 if (smp_started) {
1141 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
1142 }
1143 }
1144
1145 void
smp_masked_invlpg_range(cpuset_t mask,vm_offset_t addr1,vm_offset_t addr2)1146 smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
1147 {
1148
1149 if (smp_started) {
1150 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
1151 }
1152 }
1153
1154 /*
1155 * send an IPI to a set of cpus.
1156 */
1157 void
ipi_selected(cpuset_t cpus,u_int ipi)1158 ipi_selected(cpuset_t cpus, u_int ipi)
1159 {
1160 int cpu;
1161
1162 /*
1163 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1164 * of help in order to understand what is the source.
1165 * Set the mask of receiving CPUs for this purpose.
1166 */
1167 if (ipi == IPI_STOP_HARD)
1168 CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus);
1169
1170 while ((cpu = CPU_FFS(&cpus)) != 0) {
1171 cpu--;
1172 CPU_CLR(cpu, &cpus);
1173 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1174 ipi_send_cpu(cpu, ipi);
1175 }
1176 }
1177
1178 /*
1179 * send an IPI to a specific CPU.
1180 */
1181 void
ipi_cpu(int cpu,u_int ipi)1182 ipi_cpu(int cpu, u_int ipi)
1183 {
1184
1185 /*
1186 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1187 * of help in order to understand what is the source.
1188 * Set the mask of receiving CPUs for this purpose.
1189 */
1190 if (ipi == IPI_STOP_HARD)
1191 CPU_SET_ATOMIC(cpu, &ipi_nmi_pending);
1192
1193 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1194 ipi_send_cpu(cpu, ipi);
1195 }
1196
1197 /*
1198 * send an IPI to all CPUs EXCEPT myself
1199 */
1200 void
ipi_all_but_self(u_int ipi)1201 ipi_all_but_self(u_int ipi)
1202 {
1203 cpuset_t other_cpus;
1204
1205 /*
1206 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1207 * of help in order to understand what is the source.
1208 * Set the mask of receiving CPUs for this purpose.
1209 */
1210 other_cpus = all_cpus;
1211 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
1212 if (ipi == IPI_STOP_HARD)
1213 CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus);
1214
1215 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1216 ipi_selected(other_cpus, ipi);
1217 }
1218
1219 int
ipi_nmi_handler()1220 ipi_nmi_handler()
1221 {
1222 u_int cpuid;
1223
1224 /*
1225 * As long as there is not a simple way to know about a NMI's
1226 * source, if the bitmask for the current CPU is present in
1227 * the global pending bitword an IPI_STOP_HARD has been issued
1228 * and should be handled.
1229 */
1230 cpuid = PCPU_GET(cpuid);
1231 if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
1232 return (1);
1233
1234 CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
1235 cpustop_handler();
1236 return (0);
1237 }
1238
1239 /*
1240 * Handle an IPI_STOP by saving our current context and spinning until we
1241 * are resumed.
1242 */
1243 void
cpustop_handler(void)1244 cpustop_handler(void)
1245 {
1246 int cpu;
1247
1248 cpu = PCPU_GET(cpuid);
1249
1250 savectx(&stoppcbs[cpu]);
1251
1252 /* Indicate that we are stopped */
1253 CPU_SET_ATOMIC(cpu, &stopped_cpus);
1254
1255 /* Wait for restart */
1256 while (!CPU_ISSET(cpu, &started_cpus))
1257 ia32_pause();
1258
1259 CPU_CLR_ATOMIC(cpu, &started_cpus);
1260 CPU_CLR_ATOMIC(cpu, &stopped_cpus);
1261
1262 if (cpu == 0 && cpustop_restartfunc != NULL) {
1263 cpustop_restartfunc();
1264 cpustop_restartfunc = NULL;
1265 }
1266 }
1267
1268 /*
1269 * Handlers for TLB related IPIs
1270 *
1271 * On i386 Xen PV this are no-ops since this port doesn't support SMP.
1272 */
1273 void
invltlb_handler(void)1274 invltlb_handler(void)
1275 {
1276 }
1277
1278 void
invlpg_handler(void)1279 invlpg_handler(void)
1280 {
1281 }
1282
1283 void
invlrng_handler(void)1284 invlrng_handler(void)
1285 {
1286 }
1287
1288 void
invlcache_handler(void)1289 invlcache_handler(void)
1290 {
1291 }
1292
1293 /*
1294 * This is called once the rest of the system is up and running and we're
1295 * ready to let the AP's out of the pen.
1296 */
1297 static void
release_aps(void * dummy __unused)1298 release_aps(void *dummy __unused)
1299 {
1300
1301 if (mp_ncpus == 1)
1302 return;
1303 atomic_store_rel_int(&aps_ready, 1);
1304 while (smp_started == 0)
1305 ia32_pause();
1306 }
1307 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
1308 SYSINIT(start_ipis, SI_SUB_SMP, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
1309 SYSINIT(start_cpu, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_setup_cpus, NULL);
1310