1 /*-
2 * Copyright (c) 1996, by Steve Passe
3 * Copyright (c) 2003, by Peter Wemm
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. The name of the developer may NOT be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD: stable/12/sys/x86/x86/mp_x86.c 371751 2022-03-19 00:32:33Z git2svn $");
29
30 #ifdef __i386__
31 #include "opt_apic.h"
32 #endif
33 #include "opt_cpu.h"
34 #include "opt_kstack_pages.h"
35 #include "opt_pmap.h"
36 #include "opt_sched.h"
37 #include "opt_smp.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/bus.h>
42 #include <sys/cons.h> /* cngetc() */
43 #include <sys/cpuset.h>
44 #ifdef GPROF
45 #include <sys/gmon.h>
46 #endif
47 #include <sys/interrupt.h>
48 #include <sys/kdb.h>
49 #include <sys/kernel.h>
50 #include <sys/ktr.h>
51 #include <sys/lock.h>
52 #include <sys/malloc.h>
53 #include <sys/memrange.h>
54 #include <sys/mutex.h>
55 #include <sys/pcpu.h>
56 #include <sys/proc.h>
57 #include <sys/sched.h>
58 #include <sys/smp.h>
59 #include <sys/sysctl.h>
60
61 #include <vm/vm.h>
62 #include <vm/vm_param.h>
63 #include <vm/pmap.h>
64 #include <vm/vm_kern.h>
65 #include <vm/vm_extern.h>
66 #include <vm/vm_map.h>
67
68 #include <x86/apicreg.h>
69 #include <machine/clock.h>
70 #include <machine/cpu.h>
71 #include <machine/cputypes.h>
72 #include <x86/mca.h>
73 #include <machine/md_var.h>
74 #include <machine/pcb.h>
75 #include <machine/psl.h>
76 #include <machine/smp.h>
77 #include <machine/specialreg.h>
78 #include <x86/ucode.h>
79
80 static MALLOC_DEFINE(M_CPUS, "cpus", "CPU items");
81
82 /* lock region used by kernel profiling */
83 int mcount_lock;
84
85 int mp_naps; /* # of Applications processors */
86 int boot_cpu_id = -1; /* designated BSP */
87
88 /* AP uses this during bootstrap. Do not staticize. */
89 char *bootSTK;
90 int bootAP;
91
92 /* Free these after use */
93 void *bootstacks[MAXCPU];
94 void *dpcpu;
95
96 struct pcb stoppcbs[MAXCPU];
97 struct susppcb **susppcbs;
98
99 #ifdef COUNT_IPIS
100 /* Interrupt counts. */
101 static u_long *ipi_preempt_counts[MAXCPU];
102 static u_long *ipi_ast_counts[MAXCPU];
103 u_long *ipi_invltlb_counts[MAXCPU];
104 u_long *ipi_invlrng_counts[MAXCPU];
105 u_long *ipi_invlpg_counts[MAXCPU];
106 u_long *ipi_invlcache_counts[MAXCPU];
107 u_long *ipi_rendezvous_counts[MAXCPU];
108 static u_long *ipi_hardclock_counts[MAXCPU];
109 #endif
110
111 /* Default cpu_ops implementation. */
112 struct cpu_ops cpu_ops;
113
114 /*
115 * Local data and functions.
116 */
117
118 static volatile cpuset_t ipi_stop_nmi_pending;
119
120 volatile cpuset_t resuming_cpus;
121 volatile cpuset_t toresume_cpus;
122
123 /* used to hold the AP's until we are ready to release them */
124 struct mtx ap_boot_mtx;
125
126 /* Set to 1 once we're ready to let the APs out of the pen. */
127 volatile int aps_ready = 0;
128
129 /*
130 * Store data from cpu_add() until later in the boot when we actually setup
131 * the APs.
132 */
133 struct cpu_info *cpu_info;
134 int *apic_cpuids;
135 int cpu_apic_ids[MAXCPU];
136 _Static_assert(MAXCPU <= MAX_APIC_ID,
137 "MAXCPU cannot be larger that MAX_APIC_ID");
138 _Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
139 "xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
140
141 /* Holds pending bitmap based IPIs per CPU */
142 volatile u_int cpu_ipi_pending[MAXCPU];
143
144 static void release_aps(void *dummy);
145 static void cpustop_handler_post(u_int cpu);
146
147 static int hyperthreading_allowed = 1;
148 SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_allowed, CTLFLAG_RDTUN,
149 &hyperthreading_allowed, 0, "Use Intel HTT logical CPUs");
150
151 static struct topo_node topo_root;
152
153 static int pkg_id_shift;
154 static int node_id_shift;
155 static int core_id_shift;
156 static int disabled_cpus;
157
158 struct cache_info {
159 int id_shift;
160 int present;
161 } static caches[MAX_CACHE_LEVELS];
162
163 unsigned int boot_address;
164
165 #define MiB(v) (v ## ULL << 20)
166
167 void
mem_range_AP_init(void)168 mem_range_AP_init(void)
169 {
170
171 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP)
172 mem_range_softc.mr_op->initAP(&mem_range_softc);
173 }
174
175 /*
176 * Round up to the next power of two, if necessary, and then
177 * take log2.
178 * Returns -1 if argument is zero.
179 */
180 static __inline int
mask_width(u_int x)181 mask_width(u_int x)
182 {
183
184 return (fls(x << (1 - powerof2(x))) - 1);
185 }
186
187 /*
188 * Add a cache level to the cache topology description.
189 */
190 static int
add_deterministic_cache(int type,int level,int share_count)191 add_deterministic_cache(int type, int level, int share_count)
192 {
193
194 if (type == 0)
195 return (0);
196 if (type > 3) {
197 printf("unexpected cache type %d\n", type);
198 return (1);
199 }
200 if (type == 2) /* ignore instruction cache */
201 return (1);
202 if (level == 0 || level > MAX_CACHE_LEVELS) {
203 printf("unexpected cache level %d\n", level);
204 return (1);
205 }
206
207 if (caches[level - 1].present) {
208 printf("WARNING: multiple entries for L%u data cache\n", level);
209 printf("%u => %u\n", caches[level - 1].id_shift,
210 mask_width(share_count));
211 }
212 caches[level - 1].id_shift = mask_width(share_count);
213 caches[level - 1].present = 1;
214
215 if (caches[level - 1].id_shift > pkg_id_shift) {
216 printf("WARNING: L%u data cache covers more "
217 "APIC IDs than a package (%u > %u)\n", level,
218 caches[level - 1].id_shift, pkg_id_shift);
219 caches[level - 1].id_shift = pkg_id_shift;
220 }
221 if (caches[level - 1].id_shift < core_id_shift) {
222 printf("WARNING: L%u data cache covers fewer "
223 "APIC IDs than a core (%u < %u)\n", level,
224 caches[level - 1].id_shift, core_id_shift);
225 caches[level - 1].id_shift = core_id_shift;
226 }
227
228 return (1);
229 }
230
231 /*
232 * Determine topology of processing units and caches for AMD CPUs.
233 * See:
234 * - AMD CPUID Specification (Publication # 25481)
235 * - BKDG for AMD NPT Family 0Fh Processors (Publication # 32559)
236 * - BKDG For AMD Family 10h Processors (Publication # 31116)
237 * - BKDG For AMD Family 15h Models 00h-0Fh Processors (Publication # 42301)
238 * - BKDG For AMD Family 16h Models 00h-0Fh Processors (Publication # 48751)
239 * - PPR For AMD Family 17h Models 00h-0Fh Processors (Publication # 54945)
240 */
241 static void
topo_probe_amd(void)242 topo_probe_amd(void)
243 {
244 u_int p[4];
245 uint64_t v;
246 int level;
247 int nodes_per_socket;
248 int share_count;
249 int type;
250 int i;
251
252 /* No multi-core capability. */
253 if ((amd_feature2 & AMDID2_CMP) == 0)
254 return;
255
256 /* For families 10h and newer. */
257 pkg_id_shift = (cpu_procinfo2 & AMDID_COREID_SIZE) >>
258 AMDID_COREID_SIZE_SHIFT;
259
260 /* For 0Fh family. */
261 if (pkg_id_shift == 0)
262 pkg_id_shift =
263 mask_width((cpu_procinfo2 & AMDID_CMP_CORES) + 1);
264
265 /*
266 * Families prior to 16h define the following value as
267 * cores per compute unit and we don't really care about the AMD
268 * compute units at the moment. Perhaps we should treat them as
269 * cores and cores within the compute units as hardware threads,
270 * but that's up for debate.
271 * Later families define the value as threads per compute unit,
272 * so we are following AMD's nomenclature here.
273 */
274 if ((amd_feature2 & AMDID2_TOPOLOGY) != 0 &&
275 CPUID_TO_FAMILY(cpu_id) >= 0x16) {
276 cpuid_count(0x8000001e, 0, p);
277 share_count = ((p[1] >> 8) & 0xff) + 1;
278 core_id_shift = mask_width(share_count);
279
280 /*
281 * For Zen (17h), gather Nodes per Processor. Each node is a
282 * Zeppelin die; TR and EPYC CPUs will have multiple dies per
283 * package. Communication latency between dies is higher than
284 * within them.
285 */
286 nodes_per_socket = ((p[2] >> 8) & 0x7) + 1;
287 node_id_shift = pkg_id_shift - mask_width(nodes_per_socket);
288 }
289
290 if ((amd_feature2 & AMDID2_TOPOLOGY) != 0) {
291 for (i = 0; ; i++) {
292 cpuid_count(0x8000001d, i, p);
293 type = p[0] & 0x1f;
294 level = (p[0] >> 5) & 0x7;
295 share_count = 1 + ((p[0] >> 14) & 0xfff);
296
297 if (!add_deterministic_cache(type, level, share_count))
298 break;
299 }
300 } else {
301 if (cpu_exthigh >= 0x80000005) {
302 cpuid_count(0x80000005, 0, p);
303 if (((p[2] >> 24) & 0xff) != 0) {
304 caches[0].id_shift = 0;
305 caches[0].present = 1;
306 }
307 }
308 if (cpu_exthigh >= 0x80000006) {
309 cpuid_count(0x80000006, 0, p);
310 if (((p[2] >> 16) & 0xffff) != 0) {
311 caches[1].id_shift = 0;
312 caches[1].present = 1;
313 }
314 if (((p[3] >> 18) & 0x3fff) != 0) {
315 nodes_per_socket = 1;
316 if ((amd_feature2 & AMDID2_NODE_ID) != 0) {
317 /*
318 * Handle multi-node processors that
319 * have multiple chips, each with its
320 * own L3 cache, on the same die.
321 */
322 v = rdmsr(0xc001100c);
323 nodes_per_socket = 1 + ((v >> 3) & 0x7);
324 }
325 caches[2].id_shift =
326 pkg_id_shift - mask_width(nodes_per_socket);
327 caches[2].present = 1;
328 }
329 }
330 }
331 }
332
333 /*
334 * Determine topology of processing units for Intel CPUs
335 * using CPUID Leaf 1 and Leaf 4, if supported.
336 * See:
337 * - Intel 64 Architecture Processor Topology Enumeration
338 * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
339 * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS
340 * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS
341 */
342 static void
topo_probe_intel_0x4(void)343 topo_probe_intel_0x4(void)
344 {
345 u_int p[4];
346 int max_cores;
347 int max_logical;
348
349 /* Both zero and one here mean one logical processor per package. */
350 max_logical = (cpu_feature & CPUID_HTT) != 0 ?
351 (cpu_procinfo & CPUID_HTT_CORES) >> 16 : 1;
352 if (max_logical <= 1)
353 return;
354
355 if (cpu_high >= 0x4) {
356 cpuid_count(0x04, 0, p);
357 max_cores = ((p[0] >> 26) & 0x3f) + 1;
358 } else
359 max_cores = 1;
360
361 core_id_shift = mask_width(max_logical/max_cores);
362 KASSERT(core_id_shift >= 0,
363 ("intel topo: max_cores > max_logical\n"));
364 pkg_id_shift = core_id_shift + mask_width(max_cores);
365 }
366
367 /*
368 * Determine topology of processing units for Intel CPUs
369 * using CPUID Leaf 1Fh or 0Bh, if supported.
370 * See:
371 * - Intel 64 Architecture Processor Topology Enumeration
372 * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
373 * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS
374 * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS
375 */
376 static void
topo_probe_intel_0xb(void)377 topo_probe_intel_0xb(void)
378 {
379 u_int leaf;
380 u_int p[4] = { 0 };
381 int bits;
382 int type;
383 int i;
384
385 /* Prefer leaf 1Fh (V2 Extended Topology Enumeration). */
386 if (cpu_high >= 0x1f) {
387 leaf = 0x1f;
388 cpuid_count(leaf, 0, p);
389 }
390 /* Fall back to leaf 0Bh (Extended Topology Enumeration). */
391 if (p[1] == 0) {
392 leaf = 0x0b;
393 cpuid_count(leaf, 0, p);
394 }
395 /* Fall back to leaf 04h (Deterministic Cache Parameters). */
396 if (p[1] == 0) {
397 topo_probe_intel_0x4();
398 return;
399 }
400
401 /* We only support three levels for now. */
402 for (i = 0; ; i++) {
403 cpuid_count(leaf, i, p);
404
405 bits = p[0] & 0x1f;
406 type = (p[2] >> 8) & 0xff;
407
408 if (type == 0)
409 break;
410
411 if (type == CPUID_TYPE_SMT)
412 core_id_shift = bits;
413 else if (type == CPUID_TYPE_CORE)
414 pkg_id_shift = bits;
415 else if (bootverbose)
416 printf("Topology level type %d shift: %d\n", type, bits);
417 }
418
419 if (pkg_id_shift < core_id_shift) {
420 printf("WARNING: core covers more APIC IDs than a package\n");
421 core_id_shift = pkg_id_shift;
422 }
423 }
424
425 /*
426 * Determine topology of caches for Intel CPUs.
427 * See:
428 * - Intel 64 Architecture Processor Topology Enumeration
429 * - Intel 64 and IA-32 Architectures Software Developer’s Manual
430 * Volume 2A: Instruction Set Reference, A-M,
431 * CPUID instruction
432 */
433 static void
topo_probe_intel_caches(void)434 topo_probe_intel_caches(void)
435 {
436 u_int p[4];
437 int level;
438 int share_count;
439 int type;
440 int i;
441
442 if (cpu_high < 0x4) {
443 /*
444 * Available cache level and sizes can be determined
445 * via CPUID leaf 2, but that requires a huge table of hardcoded
446 * values, so for now just assume L1 and L2 caches potentially
447 * shared only by HTT processing units, if HTT is present.
448 */
449 caches[0].id_shift = pkg_id_shift;
450 caches[0].present = 1;
451 caches[1].id_shift = pkg_id_shift;
452 caches[1].present = 1;
453 return;
454 }
455
456 for (i = 0; ; i++) {
457 cpuid_count(0x4, i, p);
458 type = p[0] & 0x1f;
459 level = (p[0] >> 5) & 0x7;
460 share_count = 1 + ((p[0] >> 14) & 0xfff);
461
462 if (!add_deterministic_cache(type, level, share_count))
463 break;
464 }
465 }
466
467 /*
468 * Determine topology of processing units and caches for Intel CPUs.
469 * See:
470 * - Intel 64 Architecture Processor Topology Enumeration
471 */
472 static void
topo_probe_intel(void)473 topo_probe_intel(void)
474 {
475
476 /*
477 * Note that 0x1 <= cpu_high < 4 case should be
478 * compatible with topo_probe_intel_0x4() logic when
479 * CPUID.1:EBX[23:16] > 0 (cpu_cores will be 1)
480 * or it should trigger the fallback otherwise.
481 */
482 if (cpu_high >= 0xb)
483 topo_probe_intel_0xb();
484 else if (cpu_high >= 0x1)
485 topo_probe_intel_0x4();
486
487 topo_probe_intel_caches();
488 }
489
490 /*
491 * Topology information is queried only on BSP, on which this
492 * code runs and for which it can query CPUID information.
493 * Then topology is extrapolated on all packages using an
494 * assumption that APIC ID to hardware component ID mapping is
495 * homogenious.
496 * That doesn't necesserily imply that the topology is uniform.
497 */
498 void
topo_probe(void)499 topo_probe(void)
500 {
501 static int cpu_topo_probed = 0;
502 struct x86_topo_layer {
503 int type;
504 int subtype;
505 int id_shift;
506 } topo_layers[MAX_CACHE_LEVELS + 4];
507 struct topo_node *parent;
508 struct topo_node *node;
509 int layer;
510 int nlayers;
511 int node_id;
512 int i;
513
514 if (cpu_topo_probed)
515 return;
516
517 CPU_ZERO(&logical_cpus_mask);
518
519 if (mp_ncpus <= 1)
520 ; /* nothing */
521 else if (cpu_vendor_id == CPU_VENDOR_AMD ||
522 cpu_vendor_id == CPU_VENDOR_HYGON)
523 topo_probe_amd();
524 else if (cpu_vendor_id == CPU_VENDOR_INTEL)
525 topo_probe_intel();
526
527 KASSERT(pkg_id_shift >= core_id_shift,
528 ("bug in APIC topology discovery"));
529
530 nlayers = 0;
531 bzero(topo_layers, sizeof(topo_layers));
532
533 topo_layers[nlayers].type = TOPO_TYPE_PKG;
534 topo_layers[nlayers].id_shift = pkg_id_shift;
535 if (bootverbose)
536 printf("Package ID shift: %u\n", topo_layers[nlayers].id_shift);
537 nlayers++;
538
539 if (pkg_id_shift > node_id_shift && node_id_shift != 0) {
540 topo_layers[nlayers].type = TOPO_TYPE_GROUP;
541 topo_layers[nlayers].id_shift = node_id_shift;
542 if (bootverbose)
543 printf("Node ID shift: %u\n",
544 topo_layers[nlayers].id_shift);
545 nlayers++;
546 }
547
548 /*
549 * Consider all caches to be within a package/chip
550 * and "in front" of all sub-components like
551 * cores and hardware threads.
552 */
553 for (i = MAX_CACHE_LEVELS - 1; i >= 0; --i) {
554 if (caches[i].present) {
555 if (node_id_shift != 0)
556 KASSERT(caches[i].id_shift <= node_id_shift,
557 ("bug in APIC topology discovery"));
558 KASSERT(caches[i].id_shift <= pkg_id_shift,
559 ("bug in APIC topology discovery"));
560 KASSERT(caches[i].id_shift >= core_id_shift,
561 ("bug in APIC topology discovery"));
562
563 topo_layers[nlayers].type = TOPO_TYPE_CACHE;
564 topo_layers[nlayers].subtype = i + 1;
565 topo_layers[nlayers].id_shift = caches[i].id_shift;
566 if (bootverbose)
567 printf("L%u cache ID shift: %u\n",
568 topo_layers[nlayers].subtype,
569 topo_layers[nlayers].id_shift);
570 nlayers++;
571 }
572 }
573
574 if (pkg_id_shift > core_id_shift) {
575 topo_layers[nlayers].type = TOPO_TYPE_CORE;
576 topo_layers[nlayers].id_shift = core_id_shift;
577 if (bootverbose)
578 printf("Core ID shift: %u\n",
579 topo_layers[nlayers].id_shift);
580 nlayers++;
581 }
582
583 topo_layers[nlayers].type = TOPO_TYPE_PU;
584 topo_layers[nlayers].id_shift = 0;
585 nlayers++;
586
587 topo_init_root(&topo_root);
588 for (i = 0; i <= max_apic_id; ++i) {
589 if (!cpu_info[i].cpu_present)
590 continue;
591
592 parent = &topo_root;
593 for (layer = 0; layer < nlayers; ++layer) {
594 node_id = i >> topo_layers[layer].id_shift;
595 parent = topo_add_node_by_hwid(parent, node_id,
596 topo_layers[layer].type,
597 topo_layers[layer].subtype);
598 }
599 }
600
601 parent = &topo_root;
602 for (layer = 0; layer < nlayers; ++layer) {
603 node_id = boot_cpu_id >> topo_layers[layer].id_shift;
604 node = topo_find_node_by_hwid(parent, node_id,
605 topo_layers[layer].type,
606 topo_layers[layer].subtype);
607 topo_promote_child(node);
608 parent = node;
609 }
610
611 cpu_topo_probed = 1;
612 }
613
614 /*
615 * Assign logical CPU IDs to local APICs.
616 */
617 void
assign_cpu_ids(void)618 assign_cpu_ids(void)
619 {
620 struct topo_node *node;
621 u_int smt_mask;
622 int nhyper;
623
624 smt_mask = (1u << core_id_shift) - 1;
625
626 /*
627 * Assign CPU IDs to local APIC IDs and disable any CPUs
628 * beyond MAXCPU. CPU 0 is always assigned to the BSP.
629 */
630 mp_ncpus = 0;
631 nhyper = 0;
632 TOPO_FOREACH(node, &topo_root) {
633 if (node->type != TOPO_TYPE_PU)
634 continue;
635
636 if ((node->hwid & smt_mask) != (boot_cpu_id & smt_mask))
637 cpu_info[node->hwid].cpu_hyperthread = 1;
638
639 if (resource_disabled("lapic", node->hwid)) {
640 if (node->hwid != boot_cpu_id)
641 cpu_info[node->hwid].cpu_disabled = 1;
642 else
643 printf("Cannot disable BSP, APIC ID = %d\n",
644 node->hwid);
645 }
646
647 if (!hyperthreading_allowed &&
648 cpu_info[node->hwid].cpu_hyperthread)
649 cpu_info[node->hwid].cpu_disabled = 1;
650
651 if (mp_ncpus >= MAXCPU)
652 cpu_info[node->hwid].cpu_disabled = 1;
653
654 if (cpu_info[node->hwid].cpu_disabled) {
655 disabled_cpus++;
656 continue;
657 }
658
659 if (cpu_info[node->hwid].cpu_hyperthread)
660 nhyper++;
661
662 cpu_apic_ids[mp_ncpus] = node->hwid;
663 apic_cpuids[node->hwid] = mp_ncpus;
664 topo_set_pu_id(node, mp_ncpus);
665 mp_ncpus++;
666 }
667
668 KASSERT(mp_maxid >= mp_ncpus - 1,
669 ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid,
670 mp_ncpus));
671
672 mp_ncores = mp_ncpus - nhyper;
673 smp_threads_per_core = mp_ncpus / mp_ncores;
674 }
675
676 /*
677 * Print various information about the SMP system hardware and setup.
678 */
679 void
cpu_mp_announce(void)680 cpu_mp_announce(void)
681 {
682 struct topo_node *node;
683 const char *hyperthread;
684 struct topo_analysis topology;
685
686 printf("FreeBSD/SMP: ");
687 if (topo_analyze(&topo_root, 1, &topology)) {
688 printf("%d package(s)", topology.entities[TOPO_LEVEL_PKG]);
689 if (topology.entities[TOPO_LEVEL_GROUP] > 1)
690 printf(" x %d groups",
691 topology.entities[TOPO_LEVEL_GROUP]);
692 if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1)
693 printf(" x %d cache groups",
694 topology.entities[TOPO_LEVEL_CACHEGROUP]);
695 if (topology.entities[TOPO_LEVEL_CORE] > 0)
696 printf(" x %d core(s)",
697 topology.entities[TOPO_LEVEL_CORE]);
698 if (topology.entities[TOPO_LEVEL_THREAD] > 1)
699 printf(" x %d hardware threads",
700 topology.entities[TOPO_LEVEL_THREAD]);
701 } else {
702 printf("Non-uniform topology");
703 }
704 printf("\n");
705
706 if (disabled_cpus) {
707 printf("FreeBSD/SMP Online: ");
708 if (topo_analyze(&topo_root, 0, &topology)) {
709 printf("%d package(s)",
710 topology.entities[TOPO_LEVEL_PKG]);
711 if (topology.entities[TOPO_LEVEL_GROUP] > 1)
712 printf(" x %d groups",
713 topology.entities[TOPO_LEVEL_GROUP]);
714 if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1)
715 printf(" x %d cache groups",
716 topology.entities[TOPO_LEVEL_CACHEGROUP]);
717 if (topology.entities[TOPO_LEVEL_CORE] > 0)
718 printf(" x %d core(s)",
719 topology.entities[TOPO_LEVEL_CORE]);
720 if (topology.entities[TOPO_LEVEL_THREAD] > 1)
721 printf(" x %d hardware threads",
722 topology.entities[TOPO_LEVEL_THREAD]);
723 } else {
724 printf("Non-uniform topology");
725 }
726 printf("\n");
727 }
728
729 if (!bootverbose)
730 return;
731
732 TOPO_FOREACH(node, &topo_root) {
733 switch (node->type) {
734 case TOPO_TYPE_PKG:
735 printf("Package HW ID = %u\n", node->hwid);
736 break;
737 case TOPO_TYPE_CORE:
738 printf("\tCore HW ID = %u\n", node->hwid);
739 break;
740 case TOPO_TYPE_PU:
741 if (cpu_info[node->hwid].cpu_hyperthread)
742 hyperthread = "/HT";
743 else
744 hyperthread = "";
745
746 if (node->subtype == 0)
747 printf("\t\tCPU (AP%s): APIC ID: %u"
748 "(disabled)\n", hyperthread, node->hwid);
749 else if (node->id == 0)
750 printf("\t\tCPU0 (BSP): APIC ID: %u\n",
751 node->hwid);
752 else
753 printf("\t\tCPU%u (AP%s): APIC ID: %u\n",
754 node->id, hyperthread, node->hwid);
755 break;
756 default:
757 /* ignored */
758 break;
759 }
760 }
761 }
762
763 /*
764 * Add a scheduling group, a group of logical processors sharing
765 * a particular cache (and, thus having an affinity), to the scheduling
766 * topology.
767 * This function recursively works on lower level caches.
768 */
769 static void
x86topo_add_sched_group(struct topo_node * root,struct cpu_group * cg_root)770 x86topo_add_sched_group(struct topo_node *root, struct cpu_group *cg_root)
771 {
772 struct topo_node *node;
773 int nchildren;
774 int ncores;
775 int i;
776
777 KASSERT(root->type == TOPO_TYPE_SYSTEM || root->type == TOPO_TYPE_CACHE ||
778 root->type == TOPO_TYPE_GROUP,
779 ("x86topo_add_sched_group: bad type: %u", root->type));
780 CPU_COPY(&root->cpuset, &cg_root->cg_mask);
781 cg_root->cg_count = root->cpu_count;
782 if (root->type == TOPO_TYPE_SYSTEM)
783 cg_root->cg_level = CG_SHARE_NONE;
784 else
785 cg_root->cg_level = root->subtype;
786
787 /*
788 * Check how many core nodes we have under the given root node.
789 * If we have multiple logical processors, but not multiple
790 * cores, then those processors must be hardware threads.
791 */
792 ncores = 0;
793 node = root;
794 while (node != NULL) {
795 if (node->type != TOPO_TYPE_CORE) {
796 node = topo_next_node(root, node);
797 continue;
798 }
799
800 ncores++;
801 node = topo_next_nonchild_node(root, node);
802 }
803
804 if (cg_root->cg_level != CG_SHARE_NONE &&
805 root->cpu_count > 1 && ncores < 2)
806 cg_root->cg_flags = CG_FLAG_SMT;
807
808 /*
809 * Find out how many cache nodes we have under the given root node.
810 * We ignore cache nodes that cover all the same processors as the
811 * root node. Also, we do not descend below found cache nodes.
812 * That is, we count top-level "non-redundant" caches under the root
813 * node.
814 */
815 nchildren = 0;
816 node = root;
817 while (node != NULL) {
818 if ((node->type != TOPO_TYPE_GROUP &&
819 node->type != TOPO_TYPE_CACHE) ||
820 (root->type != TOPO_TYPE_SYSTEM &&
821 CPU_CMP(&node->cpuset, &root->cpuset) == 0)) {
822 node = topo_next_node(root, node);
823 continue;
824 }
825 nchildren++;
826 node = topo_next_nonchild_node(root, node);
827 }
828
829 /*
830 * We are not interested in nodes including only one CPU each.
831 */
832 if (nchildren == root->cpu_count)
833 return;
834
835 cg_root->cg_child = smp_topo_alloc(nchildren);
836 cg_root->cg_children = nchildren;
837
838 /*
839 * Now find again the same cache nodes as above and recursively
840 * build scheduling topologies for them.
841 */
842 node = root;
843 i = 0;
844 while (node != NULL) {
845 if ((node->type != TOPO_TYPE_GROUP &&
846 node->type != TOPO_TYPE_CACHE) ||
847 (root->type != TOPO_TYPE_SYSTEM &&
848 CPU_CMP(&node->cpuset, &root->cpuset) == 0)) {
849 node = topo_next_node(root, node);
850 continue;
851 }
852 cg_root->cg_child[i].cg_parent = cg_root;
853 x86topo_add_sched_group(node, &cg_root->cg_child[i]);
854 i++;
855 node = topo_next_nonchild_node(root, node);
856 }
857 }
858
859 /*
860 * Build the MI scheduling topology from the discovered hardware topology.
861 */
862 struct cpu_group *
cpu_topo(void)863 cpu_topo(void)
864 {
865 struct cpu_group *cg_root;
866
867 if (mp_ncpus <= 1)
868 return (smp_topo_none());
869
870 cg_root = smp_topo_alloc(1);
871 x86topo_add_sched_group(&topo_root, cg_root);
872 return (cg_root);
873 }
874
875 static void
cpu_alloc(void * dummy __unused)876 cpu_alloc(void *dummy __unused)
877 {
878 /*
879 * Dynamically allocate the arrays that depend on the
880 * maximum APIC ID.
881 */
882 cpu_info = malloc(sizeof(*cpu_info) * (max_apic_id + 1), M_CPUS,
883 M_WAITOK | M_ZERO);
884 apic_cpuids = malloc(sizeof(*apic_cpuids) * (max_apic_id + 1), M_CPUS,
885 M_WAITOK | M_ZERO);
886 }
887 SYSINIT(cpu_alloc, SI_SUB_CPU, SI_ORDER_FIRST, cpu_alloc, NULL);
888
889 /*
890 * Add a logical CPU to the topology.
891 */
892 void
cpu_add(u_int apic_id,char boot_cpu)893 cpu_add(u_int apic_id, char boot_cpu)
894 {
895
896 if (apic_id > max_apic_id) {
897 panic("SMP: APIC ID %d too high", apic_id);
898 return;
899 }
900 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %u added twice",
901 apic_id));
902 cpu_info[apic_id].cpu_present = 1;
903 if (boot_cpu) {
904 KASSERT(boot_cpu_id == -1,
905 ("CPU %u claims to be BSP, but CPU %u already is", apic_id,
906 boot_cpu_id));
907 boot_cpu_id = apic_id;
908 cpu_info[apic_id].cpu_bsp = 1;
909 }
910 if (bootverbose)
911 printf("SMP: Added CPU %u (%s)\n", apic_id, boot_cpu ? "BSP" :
912 "AP");
913 }
914
915 void
cpu_mp_setmaxid(void)916 cpu_mp_setmaxid(void)
917 {
918
919 /*
920 * mp_ncpus and mp_maxid should be already set by calls to cpu_add().
921 * If there were no calls to cpu_add() assume this is a UP system.
922 */
923 if (mp_ncpus == 0)
924 mp_ncpus = 1;
925 }
926
927 int
cpu_mp_probe(void)928 cpu_mp_probe(void)
929 {
930
931 /*
932 * Always record BSP in CPU map so that the mbuf init code works
933 * correctly.
934 */
935 CPU_SETOF(0, &all_cpus);
936 return (mp_ncpus > 1);
937 }
938
939 /* Allocate memory for the AP trampoline. */
940 void
alloc_ap_trampoline(vm_paddr_t * physmap,unsigned int * physmap_idx)941 alloc_ap_trampoline(vm_paddr_t *physmap, unsigned int *physmap_idx)
942 {
943 unsigned int i;
944 bool allocated;
945
946 allocated = false;
947 for (i = *physmap_idx; i <= *physmap_idx; i -= 2) {
948 /*
949 * Find a memory region big enough and below the 1MB boundary
950 * for the trampoline code.
951 * NB: needs to be page aligned.
952 */
953 if (physmap[i] >= MiB(1) ||
954 (trunc_page(physmap[i + 1]) - round_page(physmap[i])) <
955 round_page(bootMP_size))
956 continue;
957
958 allocated = true;
959 /*
960 * Try to steal from the end of the region to mimic previous
961 * behaviour, else fallback to steal from the start.
962 */
963 if (physmap[i + 1] < MiB(1)) {
964 boot_address = trunc_page(physmap[i + 1]);
965 if ((physmap[i + 1] - boot_address) < bootMP_size)
966 boot_address -= round_page(bootMP_size);
967 physmap[i + 1] = boot_address;
968 } else {
969 boot_address = round_page(physmap[i]);
970 physmap[i] = boot_address + round_page(bootMP_size);
971 }
972 if (physmap[i] == physmap[i + 1] && *physmap_idx != 0) {
973 memmove(&physmap[i], &physmap[i + 2],
974 sizeof(*physmap) * (*physmap_idx - i + 2));
975 *physmap_idx -= 2;
976 }
977 break;
978 }
979
980 if (!allocated) {
981 boot_address = basemem * 1024 - bootMP_size;
982 if (bootverbose)
983 printf(
984 "Cannot find enough space for the boot trampoline, placing it at %#x",
985 boot_address);
986 }
987 }
988
989 /*
990 * AP CPU's call this to initialize themselves.
991 */
992 void
init_secondary_tail(void)993 init_secondary_tail(void)
994 {
995 u_int cpuid;
996
997 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
998
999 /*
1000 * On real hardware, switch to x2apic mode if possible. Do it
1001 * after aps_ready was signalled, to avoid manipulating the
1002 * mode while BSP might still want to send some IPI to us
1003 * (second startup IPI is ignored on modern hardware etc).
1004 */
1005 lapic_xapic_mode();
1006
1007 /* Initialize the PAT MSR. */
1008 pmap_init_pat();
1009
1010 /* set up CPU registers and state */
1011 cpu_setregs();
1012
1013 /* set up SSE/NX */
1014 initializecpu();
1015
1016 /* set up FPU state on the AP */
1017 #ifdef __amd64__
1018 fpuinit();
1019 #else
1020 npxinit(false);
1021 #endif
1022
1023 if (cpu_ops.cpu_init)
1024 cpu_ops.cpu_init();
1025
1026 /* A quick check from sanity claus */
1027 cpuid = PCPU_GET(cpuid);
1028 if (PCPU_GET(apic_id) != lapic_id()) {
1029 printf("SMP: cpuid = %d\n", cpuid);
1030 printf("SMP: actual apic_id = %d\n", lapic_id());
1031 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
1032 panic("cpuid mismatch! boom!!");
1033 }
1034
1035 /* Initialize curthread. */
1036 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
1037 PCPU_SET(curthread, PCPU_GET(idlethread));
1038
1039 mtx_lock_spin(&ap_boot_mtx);
1040
1041 mca_init();
1042
1043 /* Init local apic for irq's */
1044 lapic_setup(1);
1045
1046 /* Set memory range attributes for this CPU to match the BSP */
1047 mem_range_AP_init();
1048
1049 smp_cpus++;
1050
1051 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
1052 if (bootverbose)
1053 printf("SMP: AP CPU #%d Launched!\n", cpuid);
1054 else
1055 printf("%s%d%s", smp_cpus == 2 ? "Launching APs: " : "",
1056 cpuid, smp_cpus == mp_ncpus ? "\n" : " ");
1057
1058 /* Determine if we are a logical CPU. */
1059 if (cpu_info[PCPU_GET(apic_id)].cpu_hyperthread)
1060 CPU_SET(cpuid, &logical_cpus_mask);
1061
1062 if (bootverbose)
1063 lapic_dump("AP");
1064
1065 if (smp_cpus == mp_ncpus) {
1066 /* enable IPI's, tlb shootdown, freezes etc */
1067 atomic_store_rel_int(&smp_started, 1);
1068 }
1069
1070 #ifdef __amd64__
1071 /*
1072 * Enable global pages TLB extension
1073 * This also implicitly flushes the TLB
1074 */
1075 load_cr4(rcr4() | CR4_PGE);
1076 if (pmap_pcid_enabled)
1077 load_cr4(rcr4() | CR4_PCIDE);
1078 load_ds(_udatasel);
1079 load_es(_udatasel);
1080 load_fs(_ufssel);
1081 #endif
1082
1083 mtx_unlock_spin(&ap_boot_mtx);
1084
1085 /* Wait until all the AP's are up. */
1086 while (atomic_load_acq_int(&smp_started) == 0)
1087 ia32_pause();
1088
1089 #ifndef EARLY_AP_STARTUP
1090 /* Start per-CPU event timers. */
1091 cpu_initclocks_ap();
1092 #endif
1093
1094 /*
1095 * Assert that smp_after_idle_runnable condition is reasonable.
1096 */
1097 MPASS(PCPU_GET(curpcb) == NULL);
1098
1099 sched_throw(NULL);
1100
1101 panic("scheduler returned us to %s", __func__);
1102 /* NOTREACHED */
1103 }
1104
1105 static void
smp_after_idle_runnable(void * arg __unused)1106 smp_after_idle_runnable(void *arg __unused)
1107 {
1108 struct pcpu *pc;
1109 int cpu;
1110
1111 for (cpu = 1; cpu < mp_ncpus; cpu++) {
1112 pc = pcpu_find(cpu);
1113 while (atomic_load_ptr(&pc->pc_curpcb) == NULL)
1114 cpu_spinwait();
1115 kmem_free((vm_offset_t)bootstacks[cpu], kstack_pages *
1116 PAGE_SIZE);
1117 }
1118 }
1119 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
1120 smp_after_idle_runnable, NULL);
1121
1122 /*
1123 * We tell the I/O APIC code about all the CPUs we want to receive
1124 * interrupts. If we don't want certain CPUs to receive IRQs we
1125 * can simply not tell the I/O APIC code about them in this function.
1126 * We also do not tell it about the BSP since it tells itself about
1127 * the BSP internally to work with UP kernels and on UP machines.
1128 */
1129 void
set_interrupt_apic_ids(void)1130 set_interrupt_apic_ids(void)
1131 {
1132 u_int i, apic_id;
1133
1134 for (i = 0; i < MAXCPU; i++) {
1135 apic_id = cpu_apic_ids[i];
1136 if (apic_id == -1)
1137 continue;
1138 if (cpu_info[apic_id].cpu_bsp)
1139 continue;
1140 if (cpu_info[apic_id].cpu_disabled)
1141 continue;
1142
1143 /* Don't let hyperthreads service interrupts. */
1144 if (cpu_info[apic_id].cpu_hyperthread)
1145 continue;
1146
1147 intr_add_cpu(i);
1148 }
1149 }
1150
1151
1152 #ifdef COUNT_XINVLTLB_HITS
1153 u_int xhits_gbl[MAXCPU];
1154 u_int xhits_pg[MAXCPU];
1155 u_int xhits_rng[MAXCPU];
1156 static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
1157 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
1158 sizeof(xhits_gbl), "IU", "");
1159 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
1160 sizeof(xhits_pg), "IU", "");
1161 SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
1162 sizeof(xhits_rng), "IU", "");
1163
1164 u_int ipi_global;
1165 u_int ipi_page;
1166 u_int ipi_range;
1167 u_int ipi_range_size;
1168 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
1169 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
1170 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
1171 SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
1172 0, "");
1173 #endif /* COUNT_XINVLTLB_HITS */
1174
1175 /*
1176 * Init and startup IPI.
1177 */
1178 void
ipi_startup(int apic_id,int vector)1179 ipi_startup(int apic_id, int vector)
1180 {
1181
1182 /*
1183 * This attempts to follow the algorithm described in the
1184 * Intel Multiprocessor Specification v1.4 in section B.4.
1185 * For each IPI, we allow the local APIC ~20us to deliver the
1186 * IPI. If that times out, we panic.
1187 */
1188
1189 /*
1190 * first we do an INIT IPI: this INIT IPI might be run, resetting
1191 * and running the target CPU. OR this INIT IPI might be latched (P5
1192 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1193 * ignored.
1194 */
1195 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
1196 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
1197 lapic_ipi_wait(100);
1198
1199 /* Explicitly deassert the INIT IPI. */
1200 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL |
1201 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT,
1202 apic_id);
1203
1204 DELAY(10000); /* wait ~10mS */
1205
1206 /*
1207 * next we do a STARTUP IPI: the previous INIT IPI might still be
1208 * latched, (P5 bug) this 1st STARTUP would then terminate
1209 * immediately, and the previously started INIT IPI would continue. OR
1210 * the previous INIT IPI has already run. and this STARTUP IPI will
1211 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1212 * will run.
1213 */
1214 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
1215 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
1216 vector, apic_id);
1217 if (!lapic_ipi_wait(100))
1218 panic("Failed to deliver first STARTUP IPI to APIC %d",
1219 apic_id);
1220 DELAY(200); /* wait ~200uS */
1221
1222 /*
1223 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1224 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1225 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1226 * recognized after hardware RESET or INIT IPI.
1227 */
1228 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
1229 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
1230 vector, apic_id);
1231 if (!lapic_ipi_wait(100))
1232 panic("Failed to deliver second STARTUP IPI to APIC %d",
1233 apic_id);
1234
1235 DELAY(200); /* wait ~200uS */
1236 }
1237
1238 /*
1239 * Send an IPI to specified CPU handling the bitmap logic.
1240 */
1241 void
ipi_send_cpu(int cpu,u_int ipi)1242 ipi_send_cpu(int cpu, u_int ipi)
1243 {
1244 u_int bitmap, old_pending, new_pending;
1245
1246 KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
1247
1248 if (IPI_IS_BITMAPED(ipi)) {
1249 bitmap = 1 << ipi;
1250 ipi = IPI_BITMAP_VECTOR;
1251 do {
1252 old_pending = cpu_ipi_pending[cpu];
1253 new_pending = old_pending | bitmap;
1254 } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
1255 old_pending, new_pending));
1256 if (old_pending)
1257 return;
1258 }
1259 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
1260 }
1261
1262 void
ipi_bitmap_handler(struct trapframe frame)1263 ipi_bitmap_handler(struct trapframe frame)
1264 {
1265 struct trapframe *oldframe;
1266 struct thread *td;
1267 int cpu = PCPU_GET(cpuid);
1268 u_int ipi_bitmap;
1269
1270 critical_enter();
1271 td = curthread;
1272 td->td_intr_nesting_level++;
1273 oldframe = td->td_intr_frame;
1274 td->td_intr_frame = &frame;
1275 ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
1276 if (ipi_bitmap & (1 << IPI_PREEMPT)) {
1277 #ifdef COUNT_IPIS
1278 (*ipi_preempt_counts[cpu])++;
1279 #endif
1280 sched_preempt(td);
1281 }
1282 if (ipi_bitmap & (1 << IPI_AST)) {
1283 #ifdef COUNT_IPIS
1284 (*ipi_ast_counts[cpu])++;
1285 #endif
1286 /* Nothing to do for AST */
1287 }
1288 if (ipi_bitmap & (1 << IPI_HARDCLOCK)) {
1289 #ifdef COUNT_IPIS
1290 (*ipi_hardclock_counts[cpu])++;
1291 #endif
1292 hardclockintr();
1293 }
1294 td->td_intr_frame = oldframe;
1295 td->td_intr_nesting_level--;
1296 critical_exit();
1297 }
1298
1299 /*
1300 * send an IPI to a set of cpus.
1301 */
1302 void
ipi_selected(cpuset_t cpus,u_int ipi)1303 ipi_selected(cpuset_t cpus, u_int ipi)
1304 {
1305 int cpu;
1306
1307 /*
1308 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1309 * of help in order to understand what is the source.
1310 * Set the mask of receiving CPUs for this purpose.
1311 */
1312 if (ipi == IPI_STOP_HARD)
1313 CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &cpus);
1314
1315 while ((cpu = CPU_FFS(&cpus)) != 0) {
1316 cpu--;
1317 CPU_CLR(cpu, &cpus);
1318 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1319 ipi_send_cpu(cpu, ipi);
1320 }
1321 }
1322
1323 /*
1324 * send an IPI to a specific CPU.
1325 */
1326 void
ipi_cpu(int cpu,u_int ipi)1327 ipi_cpu(int cpu, u_int ipi)
1328 {
1329
1330 /*
1331 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1332 * of help in order to understand what is the source.
1333 * Set the mask of receiving CPUs for this purpose.
1334 */
1335 if (ipi == IPI_STOP_HARD)
1336 CPU_SET_ATOMIC(cpu, &ipi_stop_nmi_pending);
1337
1338 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
1339 ipi_send_cpu(cpu, ipi);
1340 }
1341
1342 /*
1343 * send an IPI to all CPUs EXCEPT myself
1344 */
1345 void
ipi_all_but_self(u_int ipi)1346 ipi_all_but_self(u_int ipi)
1347 {
1348 cpuset_t other_cpus;
1349
1350 other_cpus = all_cpus;
1351 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
1352 if (IPI_IS_BITMAPED(ipi)) {
1353 ipi_selected(other_cpus, ipi);
1354 return;
1355 }
1356
1357 /*
1358 * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit
1359 * of help in order to understand what is the source.
1360 * Set the mask of receiving CPUs for this purpose.
1361 */
1362 if (ipi == IPI_STOP_HARD)
1363 CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus);
1364
1365 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
1366 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
1367 }
1368
1369 void
ipi_self_from_nmi(u_int vector)1370 ipi_self_from_nmi(u_int vector)
1371 {
1372
1373 lapic_ipi_vectored(vector, APIC_IPI_DEST_SELF);
1374
1375 /* Wait for IPI to finish. */
1376 if (!lapic_ipi_wait(50000)) {
1377 if (panicstr != NULL)
1378 return;
1379 else
1380 panic("APIC: IPI is stuck");
1381 }
1382 }
1383
1384 int
ipi_nmi_handler(void)1385 ipi_nmi_handler(void)
1386 {
1387 u_int cpuid;
1388
1389 /*
1390 * As long as there is not a simple way to know about a NMI's
1391 * source, if the bitmask for the current CPU is present in
1392 * the global pending bitword an IPI_STOP_HARD has been issued
1393 * and should be handled.
1394 */
1395 cpuid = PCPU_GET(cpuid);
1396 if (!CPU_ISSET(cpuid, &ipi_stop_nmi_pending))
1397 return (1);
1398
1399 CPU_CLR_ATOMIC(cpuid, &ipi_stop_nmi_pending);
1400 cpustop_handler();
1401 return (0);
1402 }
1403
1404 int nmi_kdb_lock;
1405
1406 void
nmi_call_kdb_smp(u_int type,struct trapframe * frame)1407 nmi_call_kdb_smp(u_int type, struct trapframe *frame)
1408 {
1409 int cpu;
1410 bool call_post;
1411
1412 cpu = PCPU_GET(cpuid);
1413 if (atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1)) {
1414 nmi_call_kdb(cpu, type, frame);
1415 call_post = false;
1416 } else {
1417 savectx(&stoppcbs[cpu]);
1418 CPU_SET_ATOMIC(cpu, &stopped_cpus);
1419 while (!atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1))
1420 ia32_pause();
1421 call_post = true;
1422 }
1423 atomic_store_rel_int(&nmi_kdb_lock, 0);
1424 if (call_post)
1425 cpustop_handler_post(cpu);
1426 }
1427
1428 /*
1429 * Handle an IPI_STOP by saving our current context and spinning until we
1430 * are resumed.
1431 */
1432 void
cpustop_handler(void)1433 cpustop_handler(void)
1434 {
1435 u_int cpu;
1436
1437 cpu = PCPU_GET(cpuid);
1438
1439 savectx(&stoppcbs[cpu]);
1440
1441 /* Indicate that we are stopped */
1442 CPU_SET_ATOMIC(cpu, &stopped_cpus);
1443
1444 /* Wait for restart */
1445 while (!CPU_ISSET(cpu, &started_cpus))
1446 ia32_pause();
1447
1448 cpustop_handler_post(cpu);
1449 }
1450
1451 static void
cpustop_handler_post(u_int cpu)1452 cpustop_handler_post(u_int cpu)
1453 {
1454
1455 CPU_CLR_ATOMIC(cpu, &started_cpus);
1456 CPU_CLR_ATOMIC(cpu, &stopped_cpus);
1457
1458 /*
1459 * We don't broadcast TLB invalidations to other CPUs when they are
1460 * stopped. Hence, we clear the TLB before resuming.
1461 */
1462 invltlb_glob();
1463
1464 #if defined(__amd64__) && defined(DDB)
1465 amd64_db_resume_dbreg();
1466 #endif
1467
1468 if (cpu == 0 && cpustop_restartfunc != NULL) {
1469 cpustop_restartfunc();
1470 cpustop_restartfunc = NULL;
1471 }
1472 }
1473
1474 /*
1475 * Handle an IPI_SUSPEND by saving our current context and spinning until we
1476 * are resumed.
1477 */
1478 void
cpususpend_handler(void)1479 cpususpend_handler(void)
1480 {
1481 u_int cpu;
1482
1483 mtx_assert(&smp_ipi_mtx, MA_NOTOWNED);
1484
1485 cpu = PCPU_GET(cpuid);
1486 if (savectx(&susppcbs[cpu]->sp_pcb)) {
1487 #ifdef __amd64__
1488 fpususpend(susppcbs[cpu]->sp_fpususpend);
1489 #else
1490 npxsuspend(susppcbs[cpu]->sp_fpususpend);
1491 #endif
1492 /*
1493 * suspended_cpus is cleared shortly after each AP is restarted
1494 * by a Startup IPI, so that the BSP can proceed to restarting
1495 * the next AP.
1496 *
1497 * resuming_cpus gets cleared when the AP completes
1498 * initialization after having been released by the BSP.
1499 * resuming_cpus is probably not the best name for the
1500 * variable, because it is actually a set of processors that
1501 * haven't resumed yet and haven't necessarily started resuming.
1502 *
1503 * Note that suspended_cpus is meaningful only for ACPI suspend
1504 * as it's not really used for Xen suspend since the APs are
1505 * automatically restored to the running state and the correct
1506 * context. For the same reason resumectx is never called in
1507 * that case.
1508 */
1509 CPU_SET_ATOMIC(cpu, &suspended_cpus);
1510 CPU_SET_ATOMIC(cpu, &resuming_cpus);
1511
1512 /*
1513 * Invalidate the cache after setting the global status bits.
1514 * The last AP to set its bit may end up being an Owner of the
1515 * corresponding cache line in MOESI protocol. The AP may be
1516 * stopped before the cache line is written to the main memory.
1517 */
1518 wbinvd();
1519 } else {
1520 #ifdef __amd64__
1521 fpuresume(susppcbs[cpu]->sp_fpususpend);
1522 #else
1523 npxresume(susppcbs[cpu]->sp_fpususpend);
1524 #endif
1525 pmap_init_pat();
1526 initializecpu();
1527 PCPU_SET(switchtime, 0);
1528 PCPU_SET(switchticks, ticks);
1529
1530 /* Indicate that we have restarted and restored the context. */
1531 CPU_CLR_ATOMIC(cpu, &suspended_cpus);
1532 }
1533
1534 /* Wait for resume directive */
1535 while (!CPU_ISSET(cpu, &toresume_cpus))
1536 ia32_pause();
1537
1538 /* Re-apply microcode updates. */
1539 ucode_reload();
1540
1541 #ifdef __i386__
1542 /* Finish removing the identity mapping of low memory for this AP. */
1543 invltlb_glob();
1544 #endif
1545
1546 if (cpu_ops.cpu_resume)
1547 cpu_ops.cpu_resume();
1548 #ifdef __amd64__
1549 if (vmm_resume_p)
1550 vmm_resume_p();
1551 #endif
1552
1553 /* Resume MCA and local APIC */
1554 lapic_xapic_mode();
1555 mca_resume();
1556 lapic_setup(0);
1557
1558 /* Indicate that we are resumed */
1559 CPU_CLR_ATOMIC(cpu, &resuming_cpus);
1560 CPU_CLR_ATOMIC(cpu, &suspended_cpus);
1561 CPU_CLR_ATOMIC(cpu, &toresume_cpus);
1562 }
1563
1564
1565 void
invlcache_handler(void)1566 invlcache_handler(void)
1567 {
1568 uint32_t generation;
1569
1570 #ifdef COUNT_IPIS
1571 (*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
1572 #endif /* COUNT_IPIS */
1573
1574 /*
1575 * Reading the generation here allows greater parallelism
1576 * since wbinvd is a serializing instruction. Without the
1577 * temporary, we'd wait for wbinvd to complete, then the read
1578 * would execute, then the dependent write, which must then
1579 * complete before return from interrupt.
1580 */
1581 generation = smp_tlb_generation;
1582 wbinvd();
1583 PCPU_SET(smp_tlb_done, generation);
1584 }
1585
1586 /*
1587 * Handle an IPI_SWI by waking delayed SWI thread.
1588 */
1589 void
ipi_swi_handler(struct trapframe frame)1590 ipi_swi_handler(struct trapframe frame)
1591 {
1592
1593 intr_event_handle(clk_intr_event, &frame);
1594 }
1595
1596 /*
1597 * This is called once the rest of the system is up and running and we're
1598 * ready to let the AP's out of the pen.
1599 */
1600 static void
release_aps(void * dummy __unused)1601 release_aps(void *dummy __unused)
1602 {
1603
1604 if (mp_ncpus == 1)
1605 return;
1606 atomic_store_rel_int(&aps_ready, 1);
1607 while (smp_started == 0)
1608 ia32_pause();
1609 }
1610 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
1611
1612 #ifdef COUNT_IPIS
1613 /*
1614 * Setup interrupt counters for IPI handlers.
1615 */
1616 static void
mp_ipi_intrcnt(void * dummy)1617 mp_ipi_intrcnt(void *dummy)
1618 {
1619 char buf[64];
1620 int i;
1621
1622 CPU_FOREACH(i) {
1623 snprintf(buf, sizeof(buf), "cpu%d:invltlb", i);
1624 intrcnt_add(buf, &ipi_invltlb_counts[i]);
1625 snprintf(buf, sizeof(buf), "cpu%d:invlrng", i);
1626 intrcnt_add(buf, &ipi_invlrng_counts[i]);
1627 snprintf(buf, sizeof(buf), "cpu%d:invlpg", i);
1628 intrcnt_add(buf, &ipi_invlpg_counts[i]);
1629 snprintf(buf, sizeof(buf), "cpu%d:invlcache", i);
1630 intrcnt_add(buf, &ipi_invlcache_counts[i]);
1631 snprintf(buf, sizeof(buf), "cpu%d:preempt", i);
1632 intrcnt_add(buf, &ipi_preempt_counts[i]);
1633 snprintf(buf, sizeof(buf), "cpu%d:ast", i);
1634 intrcnt_add(buf, &ipi_ast_counts[i]);
1635 snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i);
1636 intrcnt_add(buf, &ipi_rendezvous_counts[i]);
1637 snprintf(buf, sizeof(buf), "cpu%d:hardclock", i);
1638 intrcnt_add(buf, &ipi_hardclock_counts[i]);
1639 }
1640 }
1641 SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);
1642 #endif
1643
1644 /*
1645 * Flush the TLB on other CPU's
1646 */
1647
1648 /* Variables needed for SMP tlb shootdown. */
1649 vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
1650 pmap_t smp_tlb_pmap;
1651 volatile uint32_t smp_tlb_generation;
1652
1653 #ifdef __amd64__
1654 #define read_eflags() read_rflags()
1655 #endif
1656
1657 /*
1658 * Used by pmap to request invalidation of TLB or cache on local and
1659 * remote processors. Mask provides the set of remote CPUs which are
1660 * to be signalled with the IPI specified by vector. The curcpu_cb
1661 * callback is invoked on the calling CPU in a critical section while
1662 * waiting for remote CPUs to complete the operation.
1663 *
1664 * The callback function is called unconditionally on the caller's
1665 * underlying processor, even when this processor is not set in the
1666 * mask. So, the callback function must be prepared to handle such
1667 * spurious invocations.
1668 *
1669 * This function must be called with the thread pinned, and it unpins on
1670 * completion.
1671 */
1672 static void
smp_targeted_tlb_shootdown(cpuset_t mask,u_int vector,pmap_t pmap,vm_offset_t addr1,vm_offset_t addr2,smp_invl_cb_t curcpu_cb)1673 smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
1674 vm_offset_t addr1, vm_offset_t addr2, smp_invl_cb_t curcpu_cb)
1675 {
1676 cpuset_t other_cpus;
1677 volatile uint32_t *p_cpudone;
1678 uint32_t generation;
1679 int cpu;
1680
1681 #ifdef __i386__
1682 sched_pin();
1683 #endif
1684
1685 /*
1686 * It is not necessary to signal other CPUs while booting or
1687 * when in the debugger.
1688 */
1689 if (kdb_active || panicstr != NULL || !smp_started)
1690 goto local_cb;
1691
1692 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
1693
1694 /*
1695 * Check for other cpus. Return if none.
1696 */
1697 if (CPU_ISFULLSET(&mask)) {
1698 if (mp_ncpus <= 1)
1699 goto local_cb;
1700 } else {
1701 CPU_CLR(PCPU_GET(cpuid), &mask);
1702 if (CPU_EMPTY(&mask))
1703 goto local_cb;
1704 }
1705
1706 if (!(read_eflags() & PSL_I))
1707 panic("%s: interrupts disabled", __func__);
1708 mtx_lock_spin(&smp_ipi_mtx);
1709 smp_tlb_addr1 = addr1;
1710 smp_tlb_addr2 = addr2;
1711 smp_tlb_pmap = pmap;
1712 generation = ++smp_tlb_generation;
1713 if (CPU_ISFULLSET(&mask)) {
1714 ipi_all_but_self(vector);
1715 other_cpus = all_cpus;
1716 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
1717 } else {
1718 other_cpus = mask;
1719 while ((cpu = CPU_FFS(&mask)) != 0) {
1720 cpu--;
1721 CPU_CLR(cpu, &mask);
1722 CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
1723 cpu, vector);
1724 ipi_send_cpu(cpu, vector);
1725 }
1726 }
1727 curcpu_cb(pmap, addr1, addr2);
1728 while ((cpu = CPU_FFS(&other_cpus)) != 0) {
1729 cpu--;
1730 CPU_CLR(cpu, &other_cpus);
1731 p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done;
1732 while (*p_cpudone != generation)
1733 ia32_pause();
1734 }
1735
1736 /*
1737 * Unpin before unlocking smp_ipi_mtx. If the thread owes
1738 * preemption, this allows scheduler to select thread on any
1739 * CPU from its cpuset.
1740 */
1741 sched_unpin();
1742 mtx_unlock_spin(&smp_ipi_mtx);
1743
1744 return;
1745
1746 local_cb:
1747 critical_enter();
1748 curcpu_cb(pmap, addr1, addr2);
1749 sched_unpin();
1750 critical_exit();
1751 }
1752
1753 void
smp_masked_invltlb(cpuset_t mask,pmap_t pmap,smp_invl_cb_t curcpu_cb)1754 smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)
1755 {
1756
1757 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0, curcpu_cb);
1758 #ifdef COUNT_XINVLTLB_HITS
1759 ipi_global++;
1760 #endif
1761 }
1762
1763 void
smp_masked_invlpg(cpuset_t mask,vm_offset_t addr,pmap_t pmap,smp_invl_cb_t curcpu_cb)1764 smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap,
1765 smp_invl_cb_t curcpu_cb)
1766 {
1767
1768 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, pmap, addr, 0, curcpu_cb);
1769 #ifdef COUNT_XINVLTLB_HITS
1770 ipi_page++;
1771 #endif
1772 }
1773
1774 void
smp_masked_invlpg_range(cpuset_t mask,vm_offset_t addr1,vm_offset_t addr2,pmap_t pmap,smp_invl_cb_t curcpu_cb)1775 smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2,
1776 pmap_t pmap, smp_invl_cb_t curcpu_cb)
1777 {
1778
1779 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, pmap, addr1, addr2,
1780 curcpu_cb);
1781 #ifdef COUNT_XINVLTLB_HITS
1782 ipi_range++;
1783 ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
1784 #endif
1785 }
1786
1787 void
smp_cache_flush(smp_invl_cb_t curcpu_cb)1788 smp_cache_flush(smp_invl_cb_t curcpu_cb)
1789 {
1790
1791 smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL, 0, 0,
1792 curcpu_cb);
1793 }
1794
1795 /*
1796 * Handlers for TLB related IPIs
1797 */
1798 void
invltlb_handler(void)1799 invltlb_handler(void)
1800 {
1801 uint32_t generation;
1802
1803 #ifdef COUNT_XINVLTLB_HITS
1804 xhits_gbl[PCPU_GET(cpuid)]++;
1805 #endif /* COUNT_XINVLTLB_HITS */
1806 #ifdef COUNT_IPIS
1807 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
1808 #endif /* COUNT_IPIS */
1809
1810 /*
1811 * Reading the generation here allows greater parallelism
1812 * since invalidating the TLB is a serializing operation.
1813 */
1814 generation = smp_tlb_generation;
1815 if (smp_tlb_pmap == kernel_pmap)
1816 invltlb_glob();
1817 #ifdef __amd64__
1818 else
1819 invltlb();
1820 #endif
1821 PCPU_SET(smp_tlb_done, generation);
1822 }
1823
1824 void
invlpg_handler(void)1825 invlpg_handler(void)
1826 {
1827 uint32_t generation;
1828
1829 #ifdef COUNT_XINVLTLB_HITS
1830 xhits_pg[PCPU_GET(cpuid)]++;
1831 #endif /* COUNT_XINVLTLB_HITS */
1832 #ifdef COUNT_IPIS
1833 (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
1834 #endif /* COUNT_IPIS */
1835
1836 generation = smp_tlb_generation; /* Overlap with serialization */
1837 #ifdef __i386__
1838 if (smp_tlb_pmap == kernel_pmap)
1839 #endif
1840 invlpg(smp_tlb_addr1);
1841 PCPU_SET(smp_tlb_done, generation);
1842 }
1843
1844 void
invlrng_handler(void)1845 invlrng_handler(void)
1846 {
1847 vm_offset_t addr, addr2;
1848 uint32_t generation;
1849
1850 #ifdef COUNT_XINVLTLB_HITS
1851 xhits_rng[PCPU_GET(cpuid)]++;
1852 #endif /* COUNT_XINVLTLB_HITS */
1853 #ifdef COUNT_IPIS
1854 (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
1855 #endif /* COUNT_IPIS */
1856
1857 addr = smp_tlb_addr1;
1858 addr2 = smp_tlb_addr2;
1859 generation = smp_tlb_generation; /* Overlap with serialization */
1860 #ifdef __i386__
1861 if (smp_tlb_pmap == kernel_pmap)
1862 #endif
1863 do {
1864 invlpg(addr);
1865 addr += PAGE_SIZE;
1866 } while (addr < addr2);
1867
1868 PCPU_SET(smp_tlb_done, generation);
1869 }
1870