1 /* $OpenBSD: cpu.c,v 1.137 2025/02/02 13:36:09 kettenis Exp $ */
2
3 /*
4 * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
5 * Copyright (c) 2017 Mark Kettenis <kettenis@openbsd.org>
6 *
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 #include "kstat.h"
21
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/proc.h>
25 #include <sys/malloc.h>
26 #include <sys/device.h>
27 #include <sys/sysctl.h>
28 #include <sys/task.h>
29 #include <sys/user.h>
30 #include <sys/kstat.h>
31
32 #include <uvm/uvm_extern.h>
33
34 #include <machine/fdt.h>
35 #include <machine/elf.h>
36
37 #include <dev/ofw/openfirm.h>
38 #include <dev/ofw/ofw_clock.h>
39 #include <dev/ofw/ofw_regulator.h>
40 #include <dev/ofw/ofw_thermal.h>
41 #include <dev/ofw/fdt.h>
42
43 #include <machine/cpufunc.h>
44
45 #include "psci.h"
46 #if NPSCI > 0
47 #include <dev/fdt/pscivar.h>
48 #endif
49
50 /* CPU Identification */
51 #define CPU_IMPL_ARM 0x41
52 #define CPU_IMPL_CAVIUM 0x43
53 #define CPU_IMPL_AMCC 0x50
54 #define CPU_IMPL_QCOM 0x51
55 #define CPU_IMPL_APPLE 0x61
56 #define CPU_IMPL_AMPERE 0xc0
57
58 /* ARM */
59 #define CPU_PART_CORTEX_A34 0xd02
60 #define CPU_PART_CORTEX_A53 0xd03
61 #define CPU_PART_CORTEX_A35 0xd04
62 #define CPU_PART_CORTEX_A55 0xd05
63 #define CPU_PART_CORTEX_A65 0xd06
64 #define CPU_PART_CORTEX_A57 0xd07
65 #define CPU_PART_CORTEX_A72 0xd08
66 #define CPU_PART_CORTEX_A73 0xd09
67 #define CPU_PART_CORTEX_A75 0xd0a
68 #define CPU_PART_CORTEX_A76 0xd0b
69 #define CPU_PART_NEOVERSE_N1 0xd0c
70 #define CPU_PART_CORTEX_A77 0xd0d
71 #define CPU_PART_CORTEX_A76AE 0xd0e
72 #define CPU_PART_NEOVERSE_V1 0xd40
73 #define CPU_PART_CORTEX_A78 0xd41
74 #define CPU_PART_CORTEX_A78AE 0xd42
75 #define CPU_PART_CORTEX_A65AE 0xd43
76 #define CPU_PART_CORTEX_X1 0xd44
77 #define CPU_PART_CORTEX_A510 0xd46
78 #define CPU_PART_CORTEX_A710 0xd47
79 #define CPU_PART_CORTEX_X2 0xd48
80 #define CPU_PART_NEOVERSE_N2 0xd49
81 #define CPU_PART_NEOVERSE_E1 0xd4a
82 #define CPU_PART_CORTEX_A78C 0xd4b
83 #define CPU_PART_CORTEX_X1C 0xd4c
84 #define CPU_PART_CORTEX_A715 0xd4d
85 #define CPU_PART_CORTEX_X3 0xd4e
86 #define CPU_PART_NEOVERSE_V2 0xd4f
87 #define CPU_PART_CORTEX_A520 0xd80
88 #define CPU_PART_CORTEX_A720 0xd81
89 #define CPU_PART_CORTEX_X4 0xd82
90 #define CPU_PART_NEOVERSE_V3AE 0xd83
91 #define CPU_PART_NEOVERSE_V3 0xd84
92 #define CPU_PART_CORTEX_X925 0xd85
93 #define CPU_PART_CORTEX_A725 0xd87
94 #define CPU_PART_CORTEX_A520AE 0xd88
95 #define CPU_PART_CORTEX_A720AE 0xd89
96 #define CPU_PART_NEOVERSE_N3 0xd8e
97
98 /* Cavium */
99 #define CPU_PART_THUNDERX_T88 0x0a1
100 #define CPU_PART_THUNDERX_T81 0x0a2
101 #define CPU_PART_THUNDERX_T83 0x0a3
102 #define CPU_PART_THUNDERX2_T99 0x0af
103
104 /* Applied Micro */
105 #define CPU_PART_X_GENE 0x000
106
107 /* Qualcomm */
108 #define CPU_PART_ORYON 0x001
109 #define CPU_PART_KRYO400_GOLD 0x804
110 #define CPU_PART_KRYO400_SILVER 0x805
111
112 /* Apple */
113 #define CPU_PART_ICESTORM 0x022
114 #define CPU_PART_FIRESTORM 0x023
115 #define CPU_PART_ICESTORM_PRO 0x024
116 #define CPU_PART_FIRESTORM_PRO 0x025
117 #define CPU_PART_ICESTORM_MAX 0x028
118 #define CPU_PART_FIRESTORM_MAX 0x029
119 #define CPU_PART_BLIZZARD 0x032
120 #define CPU_PART_AVALANCHE 0x033
121 #define CPU_PART_BLIZZARD_PRO 0x034
122 #define CPU_PART_AVALANCHE_PRO 0x035
123 #define CPU_PART_BLIZZARD_MAX 0x038
124 #define CPU_PART_AVALANCHE_MAX 0x039
125
126 /* Ampere */
127 #define CPU_PART_AMPERE1 0xac3
128
129 #define CPU_IMPL(midr) (((midr) >> 24) & 0xff)
130 #define CPU_PART(midr) (((midr) >> 4) & 0xfff)
131 #define CPU_VAR(midr) (((midr) >> 20) & 0xf)
132 #define CPU_REV(midr) (((midr) >> 0) & 0xf)
133
134 struct cpu_cores {
135 int id;
136 char *name;
137 };
138
139 struct cpu_cores cpu_cores_none[] = {
140 { 0, NULL },
141 };
142
143 struct cpu_cores cpu_cores_arm[] = {
144 { CPU_PART_CORTEX_A34, "Cortex-A34" },
145 { CPU_PART_CORTEX_A35, "Cortex-A35" },
146 { CPU_PART_CORTEX_A53, "Cortex-A53" },
147 { CPU_PART_CORTEX_A55, "Cortex-A55" },
148 { CPU_PART_CORTEX_A57, "Cortex-A57" },
149 { CPU_PART_CORTEX_A65, "Cortex-A65" },
150 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
151 { CPU_PART_CORTEX_A72, "Cortex-A72" },
152 { CPU_PART_CORTEX_A73, "Cortex-A73" },
153 { CPU_PART_CORTEX_A75, "Cortex-A75" },
154 { CPU_PART_CORTEX_A76, "Cortex-A76" },
155 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
156 { CPU_PART_CORTEX_A77, "Cortex-A77" },
157 { CPU_PART_CORTEX_A78, "Cortex-A78" },
158 { CPU_PART_CORTEX_A78AE, "Cortex-A78AE" },
159 { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
160 { CPU_PART_CORTEX_A510, "Cortex-A510" },
161 { CPU_PART_CORTEX_A520, "Cortex-A520" },
162 { CPU_PART_CORTEX_A520AE, "Cortex-A520AE" },
163 { CPU_PART_CORTEX_A710, "Cortex-A710" },
164 { CPU_PART_CORTEX_A715, "Cortex-A715" },
165 { CPU_PART_CORTEX_A720, "Cortex-A720" },
166 { CPU_PART_CORTEX_A720AE, "Cortex-A720AE" },
167 { CPU_PART_CORTEX_A725, "Cortex-A725" },
168 { CPU_PART_CORTEX_X1, "Cortex-X1" },
169 { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
170 { CPU_PART_CORTEX_X2, "Cortex-X2" },
171 { CPU_PART_CORTEX_X3, "Cortex-X3" },
172 { CPU_PART_CORTEX_X4, "Cortex-X4" },
173 { CPU_PART_CORTEX_X925, "Cortex-X925" },
174 { CPU_PART_NEOVERSE_E1, "Neoverse E1" },
175 { CPU_PART_NEOVERSE_N1, "Neoverse N1" },
176 { CPU_PART_NEOVERSE_N2, "Neoverse N2" },
177 { CPU_PART_NEOVERSE_N3, "Neoverse N3" },
178 { CPU_PART_NEOVERSE_V1, "Neoverse V1" },
179 { CPU_PART_NEOVERSE_V2, "Neoverse V2" },
180 { CPU_PART_NEOVERSE_V3, "Neoverse V3" },
181 { CPU_PART_NEOVERSE_V3AE, "Neoverse V3AE" },
182 { 0, NULL },
183 };
184
185 struct cpu_cores cpu_cores_cavium[] = {
186 { CPU_PART_THUNDERX_T88, "ThunderX T88" },
187 { CPU_PART_THUNDERX_T81, "ThunderX T81" },
188 { CPU_PART_THUNDERX_T83, "ThunderX T83" },
189 { CPU_PART_THUNDERX2_T99, "ThunderX2 T99" },
190 { 0, NULL },
191 };
192
193 struct cpu_cores cpu_cores_amcc[] = {
194 { CPU_PART_X_GENE, "X-Gene" },
195 { 0, NULL },
196 };
197
198 struct cpu_cores cpu_cores_qcom[] = {
199 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
200 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
201 { CPU_PART_ORYON, "Oryon" },
202 { 0, NULL },
203 };
204
205 struct cpu_cores cpu_cores_apple[] = {
206 { CPU_PART_ICESTORM, "Icestorm" },
207 { CPU_PART_FIRESTORM, "Firestorm" },
208 { CPU_PART_ICESTORM_PRO, "Icestorm Pro" },
209 { CPU_PART_FIRESTORM_PRO, "Firestorm Pro" },
210 { CPU_PART_ICESTORM_MAX, "Icestorm Max" },
211 { CPU_PART_FIRESTORM_MAX, "Firestorm Max" },
212 { CPU_PART_BLIZZARD, "Blizzard" },
213 { CPU_PART_AVALANCHE, "Avalanche" },
214 { CPU_PART_BLIZZARD_PRO, "Blizzard Pro" },
215 { CPU_PART_AVALANCHE_PRO, "Avalanche Pro" },
216 { CPU_PART_BLIZZARD_MAX, "Blizzard Max" },
217 { CPU_PART_AVALANCHE_MAX, "Avalanche Max" },
218 { 0, NULL },
219 };
220
221 struct cpu_cores cpu_cores_ampere[] = {
222 { CPU_PART_AMPERE1, "AmpereOne" },
223 { 0, NULL },
224 };
225
226 /* arm cores makers */
227 const struct implementers {
228 int id;
229 char *name;
230 struct cpu_cores *corelist;
231 } cpu_implementers[] = {
232 { CPU_IMPL_ARM, "ARM", cpu_cores_arm },
233 { CPU_IMPL_CAVIUM, "Cavium", cpu_cores_cavium },
234 { CPU_IMPL_AMCC, "Applied Micro", cpu_cores_amcc },
235 { CPU_IMPL_QCOM, "Qualcomm", cpu_cores_qcom },
236 { CPU_IMPL_APPLE, "Apple", cpu_cores_apple },
237 { CPU_IMPL_AMPERE, "Ampere", cpu_cores_ampere },
238 { 0, NULL },
239 };
240
241 char cpu_model[64];
242 int cpu_node;
243
244 uint64_t cpu_id_aa64isar0;
245 uint64_t cpu_id_aa64isar1;
246 uint64_t cpu_id_aa64isar2;
247 uint64_t cpu_id_aa64mmfr0;
248 uint64_t cpu_id_aa64mmfr1;
249 uint64_t cpu_id_aa64mmfr2;
250 uint64_t cpu_id_aa64pfr0;
251 uint64_t cpu_id_aa64pfr1;
252
253 int arm64_has_lse;
254 int arm64_has_rng;
255 #ifdef CRYPTO
256 int arm64_has_aes;
257 #endif
258
259 extern char trampoline_vectors_none[];
260 extern char trampoline_vectors_loop_8[];
261 extern char trampoline_vectors_loop_11[];
262 extern char trampoline_vectors_loop_24[];
263 extern char trampoline_vectors_loop_32[];
264 #if NPSCI > 0
265 extern char trampoline_vectors_psci_hvc[];
266 extern char trampoline_vectors_psci_smc[];
267 #endif
268 extern char trampoline_vectors_clrbhb[];
269
270 struct cpu_info *cpu_info_list = &cpu_info_primary;
271
272 int cpu_match(struct device *, void *, void *);
273 void cpu_attach(struct device *, struct device *, void *);
274
275 const struct cfattach cpu_ca = {
276 sizeof(struct device), cpu_match, cpu_attach
277 };
278
279 struct cfdriver cpu_cd = {
280 NULL, "cpu", DV_DULL
281 };
282
283 struct timeout cpu_rng_to;
284 void cpu_rng(void *);
285
286 void cpu_opp_init(struct cpu_info *, uint32_t);
287 void cpu_psci_init(struct cpu_info *);
288 void cpu_psci_idle_cycle(void);
289
290 void cpu_flush_bp_noop(void);
291 void cpu_flush_bp_psci(void);
292 void cpu_serror_apple(void);
293
294 #if NKSTAT > 0
295 void cpu_kstat_attach(struct cpu_info *ci);
296 void cpu_opp_kstat_attach(struct cpu_info *ci);
297 #endif
298
299 void
cpu_rng(void * arg)300 cpu_rng(void *arg)
301 {
302 struct timeout *to = arg;
303 uint64_t rndr;
304 int ret;
305
306 ret = __builtin_arm_rndrrs(&rndr);
307 if (ret)
308 ret = __builtin_arm_rndr(&rndr);
309 if (ret == 0) {
310 enqueue_randomness(rndr & 0xffffffff);
311 enqueue_randomness(rndr >> 32);
312 }
313
314 if (to)
315 timeout_add_msec(to, 1000);
316 }
317
318 /*
319 * Enable mitigation for Spectre-V2 branch target injection
320 * vulnerabilities (CVE-2017-5715).
321 */
322 void
cpu_mitigate_spectre_v2(struct cpu_info * ci)323 cpu_mitigate_spectre_v2(struct cpu_info *ci)
324 {
325 uint64_t id;
326
327 /*
328 * By default we let the firmware decide what mitigation is
329 * necessary.
330 */
331 ci->ci_flush_bp = cpu_flush_bp_psci;
332
333 /* Some specific CPUs are known not to be vulnerable. */
334 switch (CPU_IMPL(ci->ci_midr)) {
335 case CPU_IMPL_ARM:
336 switch (CPU_PART(ci->ci_midr)) {
337 case CPU_PART_CORTEX_A35:
338 case CPU_PART_CORTEX_A53:
339 case CPU_PART_CORTEX_A55:
340 /* Not vulnerable. */
341 ci->ci_flush_bp = cpu_flush_bp_noop;
342 break;
343 }
344 break;
345 case CPU_IMPL_QCOM:
346 switch (CPU_PART(ci->ci_midr)) {
347 case CPU_PART_KRYO400_SILVER:
348 /* Not vulnerable. */
349 ci->ci_flush_bp = cpu_flush_bp_noop;
350 break;
351 }
352 }
353
354 /*
355 * The architecture has been updated to explicitly tell us if
356 * we're not vulnerable to Spectre-V2.
357 */
358 id = READ_SPECIALREG(id_aa64pfr0_el1);
359 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL)
360 ci->ci_flush_bp = cpu_flush_bp_noop;
361 }
362
363 /*
364 * Enable mitigation for Spectre-BHB branch history injection
365 * vulnerabilities (CVE-2022-23960).
366 */
367 void
cpu_mitigate_spectre_bhb(struct cpu_info * ci)368 cpu_mitigate_spectre_bhb(struct cpu_info *ci)
369 {
370 uint64_t id;
371
372 /*
373 * If we know the CPU, we can add a branchy loop that cleans
374 * the BHB.
375 */
376 switch (CPU_IMPL(ci->ci_midr)) {
377 case CPU_IMPL_ARM:
378 switch (CPU_PART(ci->ci_midr)) {
379 case CPU_PART_CORTEX_A57:
380 case CPU_PART_CORTEX_A72:
381 ci->ci_trampoline_vectors =
382 (vaddr_t)trampoline_vectors_loop_8;
383 break;
384 case CPU_PART_CORTEX_A76:
385 case CPU_PART_CORTEX_A76AE:
386 case CPU_PART_CORTEX_A77:
387 case CPU_PART_NEOVERSE_N1:
388 ci->ci_trampoline_vectors =
389 (vaddr_t)trampoline_vectors_loop_24;
390 break;
391 case CPU_PART_CORTEX_A78:
392 case CPU_PART_CORTEX_A78AE:
393 case CPU_PART_CORTEX_A78C:
394 case CPU_PART_CORTEX_X1:
395 case CPU_PART_CORTEX_X2:
396 case CPU_PART_CORTEX_A710:
397 case CPU_PART_NEOVERSE_N2:
398 case CPU_PART_NEOVERSE_V1:
399 ci->ci_trampoline_vectors =
400 (vaddr_t)trampoline_vectors_loop_32;
401 break;
402 }
403 break;
404 case CPU_IMPL_AMPERE:
405 switch (CPU_PART(ci->ci_midr)) {
406 case CPU_PART_AMPERE1:
407 ci->ci_trampoline_vectors =
408 (vaddr_t)trampoline_vectors_loop_11;
409 break;
410 }
411 break;
412 }
413
414 /*
415 * If we're not using a loop, let firmware decide. This also
416 * covers the original Spectre-V2 in addition to Spectre-BHB.
417 */
418 #if NPSCI > 0
419 if (ci->ci_trampoline_vectors == (vaddr_t)trampoline_vectors_none &&
420 smccc_needs_arch_workaround_3()) {
421 ci->ci_flush_bp = cpu_flush_bp_noop;
422 if (psci_method() == PSCI_METHOD_HVC)
423 ci->ci_trampoline_vectors =
424 (vaddr_t)trampoline_vectors_psci_hvc;
425 if (psci_method() == PSCI_METHOD_SMC)
426 ci->ci_trampoline_vectors =
427 (vaddr_t)trampoline_vectors_psci_smc;
428 }
429 #endif
430
431 /* Prefer CLRBHB to mitigate Spectre-BHB. */
432 id = READ_SPECIALREG(id_aa64isar2_el1);
433 if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL)
434 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_clrbhb;
435
436 /* ECBHB tells us Spectre-BHB is mitigated. */
437 id = READ_SPECIALREG(id_aa64mmfr1_el1);
438 if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL)
439 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
440
441 /*
442 * The architecture has been updated to explicitly tell us if
443 * we're not vulnerable to Spectre-BHB.
444 */
445 id = READ_SPECIALREG(id_aa64pfr0_el1);
446 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT)
447 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
448 }
449
450 /*
451 * Enable mitigation for Spectre-V4 speculative store bypass
452 * vulnerabilities (CVE-2018-3639).
453 */
454 void
cpu_mitigate_spectre_v4(struct cpu_info * ci)455 cpu_mitigate_spectre_v4(struct cpu_info *ci)
456 {
457 uint64_t id;
458
459 switch (CPU_IMPL(ci->ci_midr)) {
460 case CPU_IMPL_ARM:
461 switch (CPU_PART(ci->ci_midr)) {
462 case CPU_PART_CORTEX_A35:
463 case CPU_PART_CORTEX_A53:
464 case CPU_PART_CORTEX_A55:
465 /* Not vulnerable. */
466 return;
467 }
468 break;
469 case CPU_IMPL_QCOM:
470 switch (CPU_PART(ci->ci_midr)) {
471 case CPU_PART_KRYO400_SILVER:
472 /* Not vulnerable. */
473 return;
474 }
475 break;
476 }
477
478 /* SSBS tells us Spectre-V4 is mitigated. */
479 id = READ_SPECIALREG(id_aa64pfr1_el1);
480 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE)
481 return;
482
483 /* Enable firmware workaround if required. */
484 smccc_enable_arch_workaround_2();
485 }
486
487 void
cpu_identify(struct cpu_info * ci)488 cpu_identify(struct cpu_info *ci)
489 {
490 static uint64_t prev_id_aa64isar0;
491 static uint64_t prev_id_aa64isar1;
492 static uint64_t prev_id_aa64isar2;
493 static uint64_t prev_id_aa64mmfr0;
494 static uint64_t prev_id_aa64mmfr1;
495 static uint64_t prev_id_aa64mmfr2;
496 static uint64_t prev_id_aa64pfr0;
497 static uint64_t prev_id_aa64pfr1;
498 uint64_t midr, impl, part;
499 uint64_t clidr, ccsidr, id;
500 uint32_t ctr, sets, ways, line;
501 const char *impl_name = NULL;
502 const char *part_name = NULL;
503 const char *il1p_name = NULL;
504 const char *sep;
505 struct cpu_cores *coreselecter = cpu_cores_none;
506 int ccidx;
507 int i;
508
509 midr = READ_SPECIALREG(midr_el1);
510 impl = CPU_IMPL(midr);
511 part = CPU_PART(midr);
512 ci->ci_midr = midr;
513
514 for (i = 0; cpu_implementers[i].name; i++) {
515 if (impl == cpu_implementers[i].id) {
516 impl_name = cpu_implementers[i].name;
517 coreselecter = cpu_implementers[i].corelist;
518 break;
519 }
520 }
521
522 for (i = 0; coreselecter[i].name; i++) {
523 if (part == coreselecter[i].id) {
524 part_name = coreselecter[i].name;
525 break;
526 }
527 }
528
529 if (impl_name && part_name) {
530 printf(" %s %s r%llup%llu", impl_name, part_name, CPU_VAR(midr),
531 CPU_REV(midr));
532
533 if (CPU_IS_PRIMARY(ci))
534 snprintf(cpu_model, sizeof(cpu_model),
535 "%s %s r%llup%llu", impl_name, part_name,
536 CPU_VAR(midr), CPU_REV(midr));
537 } else {
538 printf(" Unknown, MIDR 0x%llx", midr);
539
540 if (CPU_IS_PRIMARY(ci))
541 snprintf(cpu_model, sizeof(cpu_model), "Unknown");
542 }
543
544 /* Print cache information. */
545
546 ctr = READ_SPECIALREG(ctr_el0);
547 switch (ctr & CTR_IL1P_MASK) {
548 case CTR_IL1P_AIVIVT:
549 il1p_name = "AIVIVT ";
550 break;
551 case CTR_IL1P_VIPT:
552 il1p_name = "VIPT ";
553 break;
554 case CTR_IL1P_PIPT:
555 il1p_name = "PIPT ";
556 break;
557 }
558
559 id = READ_SPECIALREG(id_aa64mmfr2_el1);
560 clidr = READ_SPECIALREG(clidr_el1);
561 if (ID_AA64MMFR2_CCIDX(id) > ID_AA64MMFR2_CCIDX_IMPL) {
562 /* Reserved value. Don't print cache information. */
563 clidr = 0;
564 } else if (ID_AA64MMFR2_CCIDX(id) == ID_AA64MMFR2_CCIDX_IMPL) {
565 /* CCSIDR_EL1 uses the new 64-bit format. */
566 ccidx = 1;
567 } else {
568 /* CCSIDR_EL1 uses the old 32-bit format. */
569 ccidx = 0;
570 }
571 for (i = 0; i < 7; i++) {
572 if ((clidr & CLIDR_CTYPE_MASK) == 0)
573 break;
574 printf("\n%s:", ci->ci_dev->dv_xname);
575 sep = "";
576 if (clidr & CLIDR_CTYPE_INSN) {
577 WRITE_SPECIALREG(csselr_el1,
578 i << CSSELR_LEVEL_SHIFT | CSSELR_IND);
579 __asm volatile("isb");
580 ccsidr = READ_SPECIALREG(ccsidr_el1);
581 if (ccidx) {
582 sets = CCSIDR_CCIDX_SETS(ccsidr);
583 ways = CCSIDR_CCIDX_WAYS(ccsidr);
584 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
585 } else {
586 sets = CCSIDR_SETS(ccsidr);
587 ways = CCSIDR_WAYS(ccsidr);
588 line = CCSIDR_LINE_SIZE(ccsidr);
589 }
590 printf("%s %dKB %db/line %d-way L%d %sI-cache", sep,
591 (sets * ways * line) / 1024, line, ways, (i + 1),
592 il1p_name);
593 il1p_name = "";
594 sep = ",";
595 }
596 if (clidr & CLIDR_CTYPE_DATA) {
597 WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT);
598 __asm volatile("isb");
599 ccsidr = READ_SPECIALREG(ccsidr_el1);
600 if (ccidx) {
601 sets = CCSIDR_CCIDX_SETS(ccsidr);
602 ways = CCSIDR_CCIDX_WAYS(ccsidr);
603 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
604 } else {
605 sets = CCSIDR_SETS(ccsidr);
606 ways = CCSIDR_WAYS(ccsidr);
607 line = CCSIDR_LINE_SIZE(ccsidr);
608 }
609 printf("%s %dKB %db/line %d-way L%d D-cache", sep,
610 (sets * ways * line) / 1024, line, ways, (i + 1));
611 sep = ",";
612 }
613 if (clidr & CLIDR_CTYPE_UNIFIED) {
614 WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT);
615 __asm volatile("isb");
616 ccsidr = READ_SPECIALREG(ccsidr_el1);
617 if (ccidx) {
618 sets = CCSIDR_CCIDX_SETS(ccsidr);
619 ways = CCSIDR_CCIDX_WAYS(ccsidr);
620 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
621 } else {
622 sets = CCSIDR_SETS(ccsidr);
623 ways = CCSIDR_WAYS(ccsidr);
624 line = CCSIDR_LINE_SIZE(ccsidr);
625 }
626 printf("%s %dKB %db/line %d-way L%d cache", sep,
627 (sets * ways * line) / 1024, line, ways, (i + 1));
628 }
629 clidr >>= 3;
630 }
631
632 cpu_mitigate_spectre_v2(ci);
633 cpu_mitigate_spectre_bhb(ci);
634 cpu_mitigate_spectre_v4(ci);
635
636 /*
637 * Apple CPUs provide detailed information for SError.
638 */
639 if (impl == CPU_IMPL_APPLE)
640 ci->ci_serror = cpu_serror_apple;
641
642 /*
643 * Skip printing CPU features if they are identical to the
644 * previous CPU.
645 */
646 if (READ_SPECIALREG(id_aa64isar0_el1) == prev_id_aa64isar0 &&
647 READ_SPECIALREG(id_aa64isar1_el1) == prev_id_aa64isar1 &&
648 READ_SPECIALREG(id_aa64isar2_el1) == prev_id_aa64isar2 &&
649 READ_SPECIALREG(id_aa64mmfr0_el1) == prev_id_aa64mmfr0 &&
650 READ_SPECIALREG(id_aa64mmfr1_el1) == prev_id_aa64mmfr1 &&
651 READ_SPECIALREG(id_aa64mmfr2_el1) == prev_id_aa64mmfr2 &&
652 READ_SPECIALREG(id_aa64pfr0_el1) == prev_id_aa64pfr0 &&
653 READ_SPECIALREG(id_aa64pfr1_el1) == prev_id_aa64pfr1)
654 return;
655
656 /*
657 * Print CPU features encoded in the ID registers.
658 */
659
660 if (READ_SPECIALREG(id_aa64isar0_el1) != cpu_id_aa64isar0) {
661 printf("\n%s: mismatched ID_AA64ISAR0_EL1",
662 ci->ci_dev->dv_xname);
663 }
664 if (READ_SPECIALREG(id_aa64isar1_el1) != cpu_id_aa64isar1) {
665 printf("\n%s: mismatched ID_AA64ISAR1_EL1",
666 ci->ci_dev->dv_xname);
667 }
668 if (READ_SPECIALREG(id_aa64isar2_el1) != cpu_id_aa64isar2) {
669 printf("\n%s: mismatched ID_AA64ISAR2_EL1",
670 ci->ci_dev->dv_xname);
671 }
672 if (READ_SPECIALREG(id_aa64mmfr0_el1) != cpu_id_aa64mmfr0) {
673 printf("\n%s: mismatched ID_AA64MMFR0_EL1",
674 ci->ci_dev->dv_xname);
675 }
676 id = READ_SPECIALREG(id_aa64mmfr1_el1);
677 /* Allow SpecSEI to be different. */
678 id &= ~ID_AA64MMFR1_SPECSEI_MASK;
679 if (id != cpu_id_aa64mmfr1) {
680 printf("\n%s: mismatched ID_AA64MMFR1_EL1",
681 ci->ci_dev->dv_xname);
682 }
683 if (READ_SPECIALREG(id_aa64mmfr2_el1) != cpu_id_aa64mmfr2) {
684 printf("\n%s: mismatched ID_AA64MMFR2_EL1",
685 ci->ci_dev->dv_xname);
686 }
687 id = READ_SPECIALREG(id_aa64pfr0_el1);
688 /* Allow CSV2/CVS3 to be different. */
689 id &= ~ID_AA64PFR0_CSV2_MASK;
690 id &= ~ID_AA64PFR0_CSV3_MASK;
691 /* Ignore 32-bit support in all exception levels. */
692 id &= ~ID_AA64PFR0_EL0_MASK;
693 id &= ~ID_AA64PFR0_EL1_MASK;
694 id &= ~ID_AA64PFR0_EL2_MASK;
695 id &= ~ID_AA64PFR0_EL3_MASK;
696 if (id != cpu_id_aa64pfr0) {
697 printf("\n%s: mismatched ID_AA64PFR0_EL1",
698 ci->ci_dev->dv_xname);
699 }
700 if (READ_SPECIALREG(id_aa64pfr1_el1) != cpu_id_aa64pfr1) {
701 printf("\n%s: mismatched ID_AA64PFR1_EL1",
702 ci->ci_dev->dv_xname);
703 }
704
705 printf("\n%s: ", ci->ci_dev->dv_xname);
706
707 /*
708 * ID_AA64ISAR0
709 */
710 id = READ_SPECIALREG(id_aa64isar0_el1);
711 sep = "";
712
713 if (ID_AA64ISAR0_RNDR(id) >= ID_AA64ISAR0_RNDR_IMPL) {
714 printf("%sRNDR", sep);
715 sep = ",";
716 arm64_has_rng = 1;
717 }
718
719 if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IOS) {
720 printf("%sTLBIOS", sep);
721 sep = ",";
722 }
723 if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IRANGE)
724 printf("+IRANGE");
725
726 if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_BASE) {
727 printf("%sTS", sep);
728 sep = ",";
729 }
730 if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_AXFLAG)
731 printf("+AXFLAG");
732
733 if (ID_AA64ISAR0_FHM(id) >= ID_AA64ISAR0_FHM_IMPL) {
734 printf("%sFHM", sep);
735 sep = ",";
736 }
737
738 if (ID_AA64ISAR0_DP(id) >= ID_AA64ISAR0_DP_IMPL) {
739 printf("%sDP", sep);
740 sep = ",";
741 }
742
743 if (ID_AA64ISAR0_SM4(id) >= ID_AA64ISAR0_SM4_IMPL) {
744 printf("%sSM4", sep);
745 sep = ",";
746 }
747
748 if (ID_AA64ISAR0_SM3(id) >= ID_AA64ISAR0_SM3_IMPL) {
749 printf("%sSM3", sep);
750 sep = ",";
751 }
752
753 if (ID_AA64ISAR0_SHA3(id) >= ID_AA64ISAR0_SHA3_IMPL) {
754 printf("%sSHA3", sep);
755 sep = ",";
756 }
757
758 if (ID_AA64ISAR0_RDM(id) >= ID_AA64ISAR0_RDM_IMPL) {
759 printf("%sRDM", sep);
760 sep = ",";
761 }
762
763 if (ID_AA64ISAR0_ATOMIC(id) >= ID_AA64ISAR0_ATOMIC_IMPL) {
764 printf("%sAtomic", sep);
765 sep = ",";
766 arm64_has_lse = 1;
767 }
768
769 if (ID_AA64ISAR0_CRC32(id) >= ID_AA64ISAR0_CRC32_BASE) {
770 printf("%sCRC32", sep);
771 sep = ",";
772 }
773
774 if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_BASE) {
775 printf("%sSHA2", sep);
776 sep = ",";
777 }
778 if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_512)
779 printf("+SHA512");
780
781 if (ID_AA64ISAR0_SHA1(id) >= ID_AA64ISAR0_SHA1_BASE) {
782 printf("%sSHA1", sep);
783 sep = ",";
784 }
785
786 if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_BASE) {
787 printf("%sAES", sep);
788 sep = ",";
789 #ifdef CRYPTO
790 arm64_has_aes = 1;
791 #endif
792 }
793 if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_PMULL)
794 printf("+PMULL");
795
796 /*
797 * ID_AA64ISAR1
798 */
799 id = READ_SPECIALREG(id_aa64isar1_el1);
800
801 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_BASE) {
802 printf("%sLS64", sep);
803 sep = ",";
804 }
805 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_V)
806 printf("+V");
807 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_ACCDATA)
808 printf("+ACCDATA");
809
810 if (ID_AA64ISAR1_XS(id) >= ID_AA64ISAR1_XS_IMPL) {
811 printf("%sXS", sep);
812 sep = ",";
813 }
814
815 if (ID_AA64ISAR1_I8MM(id) >= ID_AA64ISAR1_I8MM_IMPL) {
816 printf("%sI8MM", sep);
817 sep = ",";
818 }
819
820 if (ID_AA64ISAR1_DGH(id) >= ID_AA64ISAR1_DGH_IMPL) {
821 printf("%sDGH", sep);
822 sep = ",";
823 }
824
825 if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_BASE) {
826 printf("%sBF16", sep);
827 sep = ",";
828 }
829 if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_EBF)
830 printf("+EBF");
831
832 if (ID_AA64ISAR1_SPECRES(id) >= ID_AA64ISAR1_SPECRES_IMPL) {
833 printf("%sSPECRES", sep);
834 sep = ",";
835 }
836
837 if (ID_AA64ISAR1_SB(id) >= ID_AA64ISAR1_SB_IMPL) {
838 printf("%sSB", sep);
839 sep = ",";
840 }
841
842 if (ID_AA64ISAR1_FRINTTS(id) >= ID_AA64ISAR1_FRINTTS_IMPL) {
843 printf("%sFRINTTS", sep);
844 sep = ",";
845 }
846
847 if (ID_AA64ISAR1_GPI(id) >= ID_AA64ISAR1_GPI_IMPL) {
848 printf("%sGPI", sep);
849 sep = ",";
850 }
851
852 if (ID_AA64ISAR1_GPA(id) >= ID_AA64ISAR1_GPA_IMPL) {
853 printf("%sGPA", sep);
854 sep = ",";
855 }
856
857 if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_BASE) {
858 printf("%sLRCPC", sep);
859 sep = ",";
860 }
861 if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_LDAPUR)
862 printf("+LDAPUR");
863
864 if (ID_AA64ISAR1_FCMA(id) >= ID_AA64ISAR1_FCMA_IMPL) {
865 printf("%sFCMA", sep);
866 sep = ",";
867 }
868
869 if (ID_AA64ISAR1_JSCVT(id) >= ID_AA64ISAR1_JSCVT_IMPL) {
870 printf("%sJSCVT", sep);
871 sep = ",";
872 }
873
874 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_PAC) {
875 printf("%sAPI", sep);
876 sep = ",";
877 }
878 if (ID_AA64ISAR1_API(id) == ID_AA64ISAR1_API_EPAC)
879 printf("+EPAC");
880 else if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_EPAC2)
881 printf("+EPAC2");
882 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC)
883 printf("+FPAC");
884 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC_COMBINED)
885 printf("+COMBINED");
886
887 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_PAC) {
888 printf("%sAPA", sep);
889 sep = ",";
890 }
891 if (ID_AA64ISAR1_APA(id) == ID_AA64ISAR1_APA_EPAC)
892 printf("+EPAC");
893 else if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_EPAC2)
894 printf("+EPAC2");
895 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC)
896 printf("+FPAC");
897 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC_COMBINED)
898 printf("+COMBINED");
899
900 if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_IMPL) {
901 printf("%sDPB", sep);
902 sep = ",";
903 }
904 if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_DCCVADP)
905 printf("+DCCVADP");
906
907 /*
908 * ID_AA64ISAR2
909 */
910 id = READ_SPECIALREG(id_aa64isar2_el1);
911
912 if (ID_AA64ISAR2_CSSC(id) >= ID_AA64ISAR2_CSSC_IMPL) {
913 printf("%sCSSC", sep);
914 sep = ",";
915 }
916
917 if (ID_AA64ISAR2_RPRFM(id) >= ID_AA64ISAR2_RPRFM_IMPL) {
918 printf("%sRPRFM", sep);
919 sep = ",";
920 }
921
922 if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL) {
923 printf("%sCLRBHB", sep);
924 sep = ",";
925 }
926
927 if (ID_AA64ISAR2_BC(id) >= ID_AA64ISAR2_BC_IMPL) {
928 printf("%sBC", sep);
929 sep = ",";
930 }
931
932 if (ID_AA64ISAR2_MOPS(id) >= ID_AA64ISAR2_MOPS_IMPL) {
933 printf("%sMOPS", sep);
934 sep = ",";
935 }
936
937 if (ID_AA64ISAR2_GPA3(id) >= ID_AA64ISAR2_GPA3_IMPL) {
938 printf("%sGPA3", sep);
939 sep = ",";
940 }
941
942 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_PAC) {
943 printf("%sAPA3", sep);
944 sep = ",";
945 }
946 if (ID_AA64ISAR2_APA3(id) == ID_AA64ISAR2_APA3_EPAC)
947 printf("+EPAC");
948 else if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_EPAC2)
949 printf("+EPAC2");
950 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC)
951 printf("+FPAC");
952 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC_COMBINED)
953 printf("+COMBINED");
954
955 if (ID_AA64ISAR2_RPRES(id) >= ID_AA64ISAR2_RPRES_IMPL) {
956 printf("%sRPRES", sep);
957 sep = ",";
958 }
959
960 if (ID_AA64ISAR2_WFXT(id) >= ID_AA64ISAR2_WFXT_IMPL) {
961 printf("%sWFXT", sep);
962 sep = ",";
963 }
964
965 /*
966 * ID_AA64MMFR0
967 *
968 * We only print ASIDBits for now.
969 */
970 id = READ_SPECIALREG(id_aa64mmfr0_el1);
971
972 if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_IMPL) {
973 printf("%sECV", sep);
974 sep = ",";
975 }
976 if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_CNTHCTL)
977 printf("+CNTHCTL");
978
979 if (ID_AA64MMFR0_ASID_BITS(id) == ID_AA64MMFR0_ASID_BITS_16) {
980 printf("%sASID16", sep);
981 sep = ",";
982 }
983
984 /*
985 * ID_AA64MMFR1
986 *
987 * We omit printing most virtualization related fields for now.
988 */
989 id = READ_SPECIALREG(id_aa64mmfr1_el1);
990
991 if (ID_AA64MMFR1_AFP(id) >= ID_AA64MMFR1_AFP_IMPL) {
992 printf("%sAFP", sep);
993 sep = ",";
994 }
995
996 if (ID_AA64MMFR1_SPECSEI(id) >= ID_AA64MMFR1_SPECSEI_IMPL) {
997 printf("%sSpecSEI", sep);
998 sep = ",";
999 }
1000
1001 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_IMPL) {
1002 printf("%sPAN", sep);
1003 sep = ",";
1004 }
1005 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_ATS1E1)
1006 printf("+ATS1E1");
1007 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_EPAN)
1008 printf("+EPAN");
1009
1010 if (ID_AA64MMFR1_LO(id) >= ID_AA64MMFR1_LO_IMPL) {
1011 printf("%sLO", sep);
1012 sep = ",";
1013 }
1014
1015 if (ID_AA64MMFR1_HPDS(id) >= ID_AA64MMFR1_HPDS_IMPL) {
1016 printf("%sHPDS", sep);
1017 sep = ",";
1018 }
1019
1020 if (ID_AA64MMFR1_VH(id) >= ID_AA64MMFR1_VH_IMPL) {
1021 printf("%sVH", sep);
1022 sep = ",";
1023 }
1024
1025 if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF) {
1026 printf("%sHAF", sep);
1027 sep = ",";
1028 }
1029 if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF_DBS)
1030 printf("DBS");
1031
1032 if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL) {
1033 printf("%sECBHB", sep);
1034 sep = ",";
1035 }
1036
1037 /*
1038 * ID_AA64MMFR2
1039 */
1040 id = READ_SPECIALREG(id_aa64mmfr2_el1);
1041
1042 if (ID_AA64MMFR2_IDS(id) >= ID_AA64MMFR2_IDS_IMPL) {
1043 printf("%sIDS", sep);
1044 sep = ",";
1045 }
1046
1047 if (ID_AA64MMFR2_AT(id) >= ID_AA64MMFR2_AT_IMPL) {
1048 printf("%sAT", sep);
1049 sep = ",";
1050 }
1051
1052 /*
1053 * ID_AA64PFR0
1054 */
1055 id = READ_SPECIALREG(id_aa64pfr0_el1);
1056
1057 if (ID_AA64PFR0_CSV3(id) >= ID_AA64PFR0_CSV3_IMPL) {
1058 printf("%sCSV3", sep);
1059 sep = ",";
1060 }
1061
1062 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL) {
1063 printf("%sCSV2", sep);
1064 sep = ",";
1065 }
1066 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_SCXT)
1067 printf("+SCXT");
1068 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT)
1069 printf("+HCXT");
1070
1071 if (ID_AA64PFR0_DIT(id) >= ID_AA64PFR0_DIT_IMPL) {
1072 printf("%sDIT", sep);
1073 sep = ",";
1074 }
1075
1076 if (ID_AA64PFR0_ADV_SIMD(id) != ID_AA64PFR0_ADV_SIMD_NONE &&
1077 ID_AA64PFR0_ADV_SIMD(id) >= ID_AA64PFR0_ADV_SIMD_HP) {
1078 printf("%sAdvSIMD+HP", sep);
1079 sep = ",";
1080 }
1081
1082 if (ID_AA64PFR0_FP(id) != ID_AA64PFR0_FP_NONE &&
1083 ID_AA64PFR0_FP(id) >= ID_AA64PFR0_FP_HP) {
1084 printf("%sFP+HP", sep);
1085 sep = ",";
1086 }
1087
1088 /*
1089 * ID_AA64PFR1
1090 */
1091 id = READ_SPECIALREG(id_aa64pfr1_el1);
1092
1093 if (ID_AA64PFR1_BT(id) >= ID_AA64PFR1_BT_IMPL) {
1094 printf("%sBT", sep);
1095 sep = ",";
1096 }
1097
1098 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE) {
1099 printf("%sSSBS", sep);
1100 sep = ",";
1101 }
1102 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE_MSR)
1103 printf("+MSR");
1104
1105 if (ID_AA64PFR1_MTE(id) >= ID_AA64PFR1_MTE_IMPL) {
1106 printf("%sMTE", sep);
1107 sep = ",";
1108 }
1109
1110 prev_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
1111 prev_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
1112 prev_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
1113 prev_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
1114 prev_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1115 prev_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1116 prev_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1117 prev_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
1118
1119 #ifdef CPU_DEBUG
1120 id = READ_SPECIALREG(id_aa64afr0_el1);
1121 printf("\nID_AA64AFR0_EL1: 0x%016llx", id);
1122 id = READ_SPECIALREG(id_aa64afr1_el1);
1123 printf("\nID_AA64AFR1_EL1: 0x%016llx", id);
1124 id = READ_SPECIALREG(id_aa64dfr0_el1);
1125 printf("\nID_AA64DFR0_EL1: 0x%016llx", id);
1126 id = READ_SPECIALREG(id_aa64dfr1_el1);
1127 printf("\nID_AA64DFR1_EL1: 0x%016llx", id);
1128 id = READ_SPECIALREG(id_aa64isar0_el1);
1129 printf("\nID_AA64ISAR0_EL1: 0x%016llx", id);
1130 id = READ_SPECIALREG(id_aa64isar1_el1);
1131 printf("\nID_AA64ISAR1_EL1: 0x%016llx", id);
1132 id = READ_SPECIALREG(id_aa64isar2_el1);
1133 printf("\nID_AA64ISAR2_EL1: 0x%016llx", id);
1134 id = READ_SPECIALREG(id_aa64mmfr0_el1);
1135 printf("\nID_AA64MMFR0_EL1: 0x%016llx", id);
1136 id = READ_SPECIALREG(id_aa64mmfr1_el1);
1137 printf("\nID_AA64MMFR1_EL1: 0x%016llx", id);
1138 id = READ_SPECIALREG(id_aa64mmfr2_el1);
1139 printf("\nID_AA64MMFR2_EL1: 0x%016llx", id);
1140 id = READ_SPECIALREG(id_aa64pfr0_el1);
1141 printf("\nID_AA64PFR0_EL1: 0x%016llx", id);
1142 id = READ_SPECIALREG(id_aa64pfr1_el1);
1143 printf("\nID_AA64PFR1_EL1: 0x%016llx", id);
1144 #endif
1145 }
1146
1147 void
cpu_identify_cleanup(void)1148 cpu_identify_cleanup(void)
1149 {
1150 uint64_t id_aa64mmfr2;
1151 uint64_t value;
1152
1153 /* ID_AA64ISAR0_EL1 */
1154 value = cpu_id_aa64isar0 & ID_AA64ISAR0_MASK;
1155 value &= ~ID_AA64ISAR0_TLB_MASK;
1156 cpu_id_aa64isar0 = value;
1157
1158 /* ID_AA64ISAR1_EL1 */
1159 value = cpu_id_aa64isar1 & ID_AA64ISAR1_MASK;
1160 value &= ~ID_AA64ISAR1_SPECRES_MASK;
1161 cpu_id_aa64isar1 = value;
1162
1163 /* ID_AA64ISAR2_EL1 */
1164 value = cpu_id_aa64isar2 & ID_AA64ISAR2_MASK;
1165 value &= ~ID_AA64ISAR2_CLRBHB_MASK;
1166 cpu_id_aa64isar2 = value;
1167
1168 /* ID_AA64MMFR0_EL1 */
1169 value = 0;
1170 value |= cpu_id_aa64mmfr0 & ID_AA64MMFR0_ECV_MASK;
1171 cpu_id_aa64mmfr0 = value;
1172
1173 /* ID_AA64MMFR1_EL1 */
1174 value = 0;
1175 value |= cpu_id_aa64mmfr1 & ID_AA64MMFR1_AFP_MASK;
1176 cpu_id_aa64mmfr1 = value;
1177
1178 /* ID_AA64MMFR2_EL1 */
1179 value = 0;
1180 value |= cpu_id_aa64mmfr2 & ID_AA64MMFR2_AT_MASK;
1181 cpu_id_aa64mmfr2 = value;
1182
1183 /* ID_AA64PFR0_EL1 */
1184 value = 0;
1185 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_FP_MASK;
1186 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_ADV_SIMD_MASK;
1187 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_DIT_MASK;
1188 cpu_id_aa64pfr0 = value;
1189
1190 /* ID_AA64PFR1_EL1 */
1191 value = 0;
1192 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_BT_MASK;
1193 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_SSBS_MASK;
1194 cpu_id_aa64pfr1 = value;
1195
1196 /* HWCAP */
1197 hwcap |= HWCAP_FP; /* OpenBSD assumes Floating-point support */
1198 hwcap |= HWCAP_ASIMD; /* OpenBSD assumes Advanced SIMD support */
1199 /* HWCAP_EVTSTRM: OpenBSD kernel doesn't configure event stream */
1200 if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_BASE)
1201 hwcap |= HWCAP_AES;
1202 if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_PMULL)
1203 hwcap |= HWCAP_PMULL;
1204 if (ID_AA64ISAR0_SHA1(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA1_BASE)
1205 hwcap |= HWCAP_SHA1;
1206 if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_BASE)
1207 hwcap |= HWCAP_SHA2;
1208 if (ID_AA64ISAR0_CRC32(cpu_id_aa64isar0) >= ID_AA64ISAR0_CRC32_BASE)
1209 hwcap |= HWCAP_CRC32;
1210 if (ID_AA64ISAR0_ATOMIC(cpu_id_aa64isar0) >= ID_AA64ISAR0_ATOMIC_IMPL)
1211 hwcap |= HWCAP_ATOMICS;
1212 if (ID_AA64PFR0_FP(cpu_id_aa64pfr0) != ID_AA64PFR0_FP_NONE &&
1213 ID_AA64PFR0_FP(cpu_id_aa64pfr0) >= ID_AA64PFR0_FP_HP)
1214 hwcap |= HWCAP_FPHP;
1215 if (ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) != ID_AA64PFR0_ADV_SIMD_NONE &&
1216 ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) >= ID_AA64PFR0_ADV_SIMD_HP)
1217 hwcap |= HWCAP_ASIMDHP;
1218 id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1219 if (ID_AA64MMFR2_IDS(id_aa64mmfr2) >= ID_AA64MMFR2_IDS_IMPL)
1220 hwcap |= HWCAP_CPUID;
1221 if (ID_AA64ISAR0_RDM(cpu_id_aa64isar0) >= ID_AA64ISAR0_RDM_IMPL)
1222 hwcap |= HWCAP_ASIMDRDM;
1223 if (ID_AA64ISAR1_JSCVT(cpu_id_aa64isar1) >= ID_AA64ISAR1_JSCVT_IMPL)
1224 hwcap |= HWCAP_JSCVT;
1225 if (ID_AA64ISAR1_FCMA(cpu_id_aa64isar1) >= ID_AA64ISAR1_FCMA_IMPL)
1226 hwcap |= HWCAP_FCMA;
1227 if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_BASE)
1228 hwcap |= HWCAP_LRCPC;
1229 if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_IMPL)
1230 hwcap |= HWCAP_DCPOP;
1231 if (ID_AA64ISAR0_SHA3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA3_IMPL)
1232 hwcap |= HWCAP_SHA3;
1233 if (ID_AA64ISAR0_SM3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM3_IMPL)
1234 hwcap |= HWCAP_SM3;
1235 if (ID_AA64ISAR0_SM4(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM4_IMPL)
1236 hwcap |= HWCAP_SM4;
1237 if (ID_AA64ISAR0_DP(cpu_id_aa64isar0) >= ID_AA64ISAR0_DP_IMPL)
1238 hwcap |= HWCAP_ASIMDDP;
1239 if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_512)
1240 hwcap |= HWCAP_SHA512;
1241 /* HWCAP_SVE: OpenBSD kernel doesn't provide SVE support */
1242 if (ID_AA64ISAR0_FHM(cpu_id_aa64isar0) >= ID_AA64ISAR0_FHM_IMPL)
1243 hwcap |= HWCAP_ASIMDFHM;
1244 if (ID_AA64PFR0_DIT(cpu_id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL)
1245 hwcap |= HWCAP_DIT;
1246 if (ID_AA64MMFR2_AT(cpu_id_aa64mmfr2) >= ID_AA64MMFR2_AT_IMPL)
1247 hwcap |= HWCAP_USCAT;
1248 if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_LDAPUR)
1249 hwcap |= HWCAP_ILRCPC;
1250 if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_BASE)
1251 hwcap |= HWCAP_FLAGM;
1252 if (ID_AA64PFR1_SSBS(cpu_id_aa64pfr1) >= ID_AA64PFR1_SSBS_PSTATE_MSR)
1253 hwcap |= HWCAP_SSBS;
1254 if (ID_AA64ISAR1_SB(cpu_id_aa64isar1) >= ID_AA64ISAR1_SB_IMPL)
1255 hwcap |= HWCAP_SB;
1256 if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC ||
1257 ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC ||
1258 ID_AA64ISAR2_APA3(cpu_id_aa64isar2) >= ID_AA64ISAR2_APA3_PAC)
1259 hwcap |= HWCAP_PACA;
1260 if (ID_AA64ISAR1_GPA(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPA_IMPL ||
1261 ID_AA64ISAR1_GPI(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPI_IMPL ||
1262 ID_AA64ISAR2_GPA3(cpu_id_aa64isar2) >= ID_AA64ISAR2_GPA3_IMPL)
1263 hwcap |= HWCAP_PACG;
1264
1265 /* HWCAP2 */
1266 if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_DCCVADP)
1267 hwcap2 |= HWCAP2_DCPODP;
1268 /* HWCAP2_SVE2: OpenBSD kernel doesn't provide SVE support */
1269 /* HWCAP2_SVEAES: OpenBSD kernel doesn't provide SVE support */
1270 /* HWCAP2_SVEPMULL: OpenBSD kernel doesn't provide SVE support */
1271 /* HWCAP2_SVEBITPERM: OpenBSD kernel doesn't provide SVE support */
1272 /* HWCAP2_SVESHA3: OpenBSD kernel doesn't provide SVE support */
1273 /* HWCAP2_SVESM4: OpenBSD kernel doesn't provide SVE support */
1274 if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_AXFLAG)
1275 hwcap2 |= HWCAP2_FLAGM2;
1276 if (ID_AA64ISAR1_FRINTTS(cpu_id_aa64isar1) >= ID_AA64ISAR1_FRINTTS_IMPL)
1277 hwcap2 |= HWCAP2_FRINT;
1278 /* HWCAP2_SVEI8MM: OpenBSD kernel doesn't provide SVE support */
1279 /* HWCAP2_SVEF32MM: OpenBSD kernel doesn't provide SVE support */
1280 /* HWCAP2_SVEF64MM: OpenBSD kernel doesn't provide SVE support */
1281 /* HWCAP2_SVEBF16: OpenBSD kernel doesn't provide SVE support */
1282 if (ID_AA64ISAR1_I8MM(cpu_id_aa64isar1) >= ID_AA64ISAR1_I8MM_IMPL)
1283 hwcap2 |= HWCAP2_I8MM;
1284 if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_BASE)
1285 hwcap2 |= HWCAP2_BF16;
1286 if (ID_AA64ISAR1_DGH(cpu_id_aa64isar1) >= ID_AA64ISAR1_DGH_IMPL)
1287 hwcap2 |= HWCAP2_DGH;
1288 if (ID_AA64ISAR0_RNDR(cpu_id_aa64isar0) >= ID_AA64ISAR0_RNDR_IMPL)
1289 hwcap2 |= HWCAP2_RNG;
1290 if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL)
1291 hwcap2 |= HWCAP2_BTI;
1292 /* HWCAP2_MTE: OpenBSD kernel doesn't provide MTE support */
1293 if (ID_AA64MMFR0_ECV(cpu_id_aa64mmfr0) >= ID_AA64MMFR0_ECV_IMPL)
1294 hwcap2 |= HWCAP2_ECV;
1295 if (ID_AA64MMFR1_AFP(cpu_id_aa64mmfr1) >= ID_AA64MMFR1_AFP_IMPL)
1296 hwcap2 |= HWCAP2_AFP;
1297 if (ID_AA64ISAR2_RPRES(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRES_IMPL)
1298 hwcap2 |= HWCAP2_RPRES;
1299 /* HWCAP2_MTE3: OpenBSD kernel doesn't provide MTE support */
1300 /* HWCAP2_SME: OpenBSD kernel doesn't provide SME support */
1301 /* HWCAP2_SME_I16I64: OpenBSD kernel doesn't provide SME support */
1302 /* HWCAP2_SME_F64F64: OpenBSD kernel doesn't provide SME support */
1303 /* HWCAP2_SME_I8I32: OpenBSD kernel doesn't provide SME support */
1304 /* HWCAP2_SME_F16F32: OpenBSD kernel doesn't provide SME support */
1305 /* HWCAP2_SME_B16F32: OpenBSD kernel doesn't provide SME support */
1306 /* HWCAP2_SME_F32F32: OpenBSD kernel doesn't provide SME support */
1307 /* HWCAP2_SME_FA64: OpenBSD kernel doesn't provide SME support */
1308 if (ID_AA64ISAR2_WFXT(cpu_id_aa64isar2) >= ID_AA64ISAR2_WFXT_IMPL)
1309 hwcap2 |= HWCAP2_WFXT;
1310 if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_EBF)
1311 hwcap2 |= HWCAP2_EBF16;
1312 /* HWCAP2_SVE_EBF16: OpenBSD kernel doesn't provide SVE support */
1313 if (ID_AA64ISAR2_CSSC(cpu_id_aa64isar2) >= ID_AA64ISAR2_CSSC_IMPL)
1314 hwcap2 |= HWCAP2_CSSC;
1315 if (ID_AA64ISAR2_RPRFM(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRFM_IMPL)
1316 hwcap2 |= HWCAP2_RPRFM;
1317 /* HWCAP2_SVE2P1: OpenBSD kernel doesn't provide SVE support */
1318 /* HWCAP2_SME2: OpenBSD kernel doesn't provide SME support */
1319 /* HWCAP2_SME2P1: OpenBSD kernel doesn't provide SME support */
1320 /* HWCAP2_SME_I16I32: OpenBSD kernel doesn't provide SME support */
1321 /* HWCAP2_SME_BI32I32: OpenBSD kernel doesn't provide SME support */
1322 /* HWCAP2_SME_B16B16: OpenBSD kernel doesn't provide SME support */
1323 /* HWCAP2_SME_F16F16: OpenBSD kernel doesn't provide SME support */
1324 if (ID_AA64ISAR2_MOPS(cpu_id_aa64isar2) >= ID_AA64ISAR2_MOPS_IMPL)
1325 hwcap2 |= HWCAP2_MOPS;
1326 if (ID_AA64ISAR2_BC(cpu_id_aa64isar2) >= ID_AA64ISAR2_BC_IMPL)
1327 hwcap2 |= HWCAP2_HBC;
1328 }
1329
1330 void cpu_init(void);
1331 int cpu_start_secondary(struct cpu_info *ci, int, uint64_t);
1332 int cpu_clockspeed(int *);
1333
1334 int
cpu_match(struct device * parent,void * cfdata,void * aux)1335 cpu_match(struct device *parent, void *cfdata, void *aux)
1336 {
1337 struct fdt_attach_args *faa = aux;
1338 uint64_t mpidr = READ_SPECIALREG(mpidr_el1);
1339 char buf[32];
1340
1341 if (OF_getprop(faa->fa_node, "device_type", buf, sizeof(buf)) <= 0 ||
1342 strcmp(buf, "cpu") != 0)
1343 return 0;
1344
1345 if (ncpus < MAXCPUS || faa->fa_reg[0].addr == (mpidr & MPIDR_AFF))
1346 return 1;
1347
1348 return 0;
1349 }
1350
1351 void
cpu_attach(struct device * parent,struct device * dev,void * aux)1352 cpu_attach(struct device *parent, struct device *dev, void *aux)
1353 {
1354 struct fdt_attach_args *faa = aux;
1355 struct cpu_info *ci;
1356 void *kstack;
1357 #ifdef MULTIPROCESSOR
1358 uint64_t mpidr = READ_SPECIALREG(mpidr_el1);
1359 #endif
1360 uint32_t opp;
1361
1362 KASSERT(faa->fa_nreg > 0);
1363
1364 #ifdef MULTIPROCESSOR
1365 if (faa->fa_reg[0].addr == (mpidr & MPIDR_AFF)) {
1366 ci = &cpu_info_primary;
1367 ci->ci_flags |= CPUF_RUNNING | CPUF_PRESENT | CPUF_PRIMARY;
1368 } else {
1369 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
1370 cpu_info[dev->dv_unit] = ci;
1371 ci->ci_next = cpu_info_list->ci_next;
1372 cpu_info_list->ci_next = ci;
1373 ci->ci_flags |= CPUF_AP;
1374 ncpus++;
1375 }
1376 #else
1377 ci = &cpu_info_primary;
1378 #endif
1379
1380 ci->ci_dev = dev;
1381 ci->ci_cpuid = dev->dv_unit;
1382 ci->ci_mpidr = faa->fa_reg[0].addr;
1383 ci->ci_node = faa->fa_node;
1384 ci->ci_self = ci;
1385
1386 printf(" mpidr %llx:", ci->ci_mpidr);
1387
1388 kstack = km_alloc(USPACE, &kv_any, &kp_zero, &kd_waitok);
1389 ci->ci_el1_stkend = (vaddr_t)kstack + USPACE - 16;
1390 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
1391
1392 #ifdef MULTIPROCESSOR
1393 if (ci->ci_flags & CPUF_AP) {
1394 char buf[32];
1395 uint64_t spinup_data = 0;
1396 int spinup_method = 0;
1397 int timeout = 10000;
1398 int len;
1399
1400 len = OF_getprop(ci->ci_node, "enable-method",
1401 buf, sizeof(buf));
1402 if (strcmp(buf, "psci") == 0) {
1403 spinup_method = 1;
1404 } else if (strcmp(buf, "spin-table") == 0) {
1405 spinup_method = 2;
1406 spinup_data = OF_getpropint64(ci->ci_node,
1407 "cpu-release-addr", 0);
1408 }
1409
1410 clockqueue_init(&ci->ci_queue);
1411 sched_init_cpu(ci);
1412 if (cpu_start_secondary(ci, spinup_method, spinup_data)) {
1413 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY);
1414 __asm volatile("dsb sy; sev" ::: "memory");
1415
1416 while ((ci->ci_flags & CPUF_IDENTIFIED) == 0 &&
1417 --timeout)
1418 delay(1000);
1419 if (timeout == 0) {
1420 printf(" failed to identify");
1421 ci->ci_flags = 0;
1422 }
1423 } else {
1424 printf(" failed to spin up");
1425 ci->ci_flags = 0;
1426 }
1427 } else {
1428 #endif
1429 cpu_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
1430 cpu_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
1431 cpu_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
1432 cpu_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
1433 cpu_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1434 cpu_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1435 cpu_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1436 cpu_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
1437
1438 /*
1439 * The SpecSEI "feature" isn't relevant for userland.
1440 * So it is fine if this field differs between CPU
1441 * cores. Mask off this field to prevent exporting it
1442 * to userland.
1443 */
1444 cpu_id_aa64mmfr1 &= ~ID_AA64MMFR1_SPECSEI_MASK;
1445
1446 /*
1447 * The CSV2/CSV3 "features" are handled on a
1448 * per-processor basis. So it is fine if these fields
1449 * differ between CPU cores. Mask off these fields to
1450 * prevent exporting these to userland.
1451 */
1452 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV2_MASK;
1453 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV3_MASK;
1454
1455 /*
1456 * We only support 64-bit mode, so we don't care about
1457 * differences in support for 32-bit mode between
1458 * cores. Mask off these fields as well.
1459 */
1460 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL0_MASK;
1461 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL1_MASK;
1462 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL2_MASK;
1463 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL3_MASK;
1464
1465 /*
1466 * Lenovo X13s ships with broken EL2 firmware that
1467 * hangs the machine if we enable PAuth.
1468 */
1469 if (hw_vendor && hw_prod && strcmp(hw_vendor, "LENOVO") == 0) {
1470 if (strncmp(hw_prod, "21BX", 4) == 0 ||
1471 strncmp(hw_prod, "21BY", 4) == 0) {
1472 cpu_id_aa64isar1 &= ~ID_AA64ISAR1_APA_MASK;
1473 cpu_id_aa64isar1 &= ~ID_AA64ISAR1_GPA_MASK;
1474 }
1475 }
1476
1477 cpu_identify(ci);
1478
1479 if (OF_getproplen(ci->ci_node, "clocks") > 0) {
1480 cpu_node = ci->ci_node;
1481 cpu_cpuspeed = cpu_clockspeed;
1482 }
1483
1484 cpu_init();
1485
1486 if (arm64_has_rng) {
1487 timeout_set(&cpu_rng_to, cpu_rng, &cpu_rng_to);
1488 cpu_rng(&cpu_rng_to);
1489 }
1490 #ifdef MULTIPROCESSOR
1491 }
1492 #endif
1493
1494 #if NKSTAT > 0
1495 cpu_kstat_attach(ci);
1496 #endif
1497
1498 opp = OF_getpropint(ci->ci_node, "operating-points-v2", 0);
1499 if (opp)
1500 cpu_opp_init(ci, opp);
1501
1502 cpu_psci_init(ci);
1503
1504 printf("\n");
1505 }
1506
1507 void
cpu_init(void)1508 cpu_init(void)
1509 {
1510 uint64_t id_aa64mmfr1, sctlr;
1511 uint64_t id_aa64pfr0;
1512 uint64_t tcr;
1513
1514 WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa);
1515 __asm volatile("isb");
1516 tcr = READ_SPECIALREG(tcr_el1);
1517 tcr &= ~TCR_T0SZ(0x3f);
1518 tcr |= TCR_T0SZ(64 - USER_SPACE_BITS);
1519 tcr |= TCR_A1;
1520 WRITE_SPECIALREG(tcr_el1, tcr);
1521 cpu_tlb_flush();
1522
1523 /* Enable PAN. */
1524 id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1525 if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_IMPL) {
1526 sctlr = READ_SPECIALREG(sctlr_el1);
1527 sctlr &= ~SCTLR_SPAN;
1528 if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_EPAN)
1529 sctlr |= SCTLR_EPAN;
1530 WRITE_SPECIALREG(sctlr_el1, sctlr);
1531 }
1532
1533 /* Enable DIT. */
1534 id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1535 if (ID_AA64PFR0_DIT(id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL)
1536 __asm volatile (".arch armv8.4-a; msr dit, #1");
1537
1538 /* Enable PAuth. */
1539 if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC ||
1540 ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC ||
1541 ID_AA64ISAR2_APA3(cpu_id_aa64isar2) >= ID_AA64ISAR2_APA3_PAC) {
1542 sctlr = READ_SPECIALREG(sctlr_el1);
1543 sctlr |= SCTLR_EnIA | SCTLR_EnDA;
1544 sctlr |= SCTLR_EnIB | SCTLR_EnDB;
1545 WRITE_SPECIALREG(sctlr_el1, sctlr);
1546 }
1547
1548 /* Enable strict BTI compatibility for PACIASP and PACIBSP. */
1549 if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL) {
1550 sctlr = READ_SPECIALREG(sctlr_el1);
1551 sctlr |= SCTLR_BT0 | SCTLR_BT1;
1552 WRITE_SPECIALREG(sctlr_el1, sctlr);
1553 }
1554
1555 /* Initialize debug registers. */
1556 WRITE_SPECIALREG(mdscr_el1, DBG_MDSCR_TDCC);
1557 WRITE_SPECIALREG(oslar_el1, 0);
1558 }
1559
1560 void
cpu_flush_bp_noop(void)1561 cpu_flush_bp_noop(void)
1562 {
1563 }
1564
1565 void
cpu_flush_bp_psci(void)1566 cpu_flush_bp_psci(void)
1567 {
1568 #if NPSCI > 0
1569 psci_flush_bp();
1570 #endif
1571 }
1572
1573 void
cpu_serror_apple(void)1574 cpu_serror_apple(void)
1575 {
1576 __asm volatile("dsb sy; isb" ::: "memory");
1577 printf("l2c_err_sts 0x%llx\n", READ_SPECIALREG(s3_3_c15_c8_0));
1578 printf("l2c_err_adr 0x%llx\n", READ_SPECIALREG(s3_3_c15_c9_0));
1579 printf("l2c_err_inf 0x%llx\n", READ_SPECIALREG(s3_3_c15_c10_0));
1580 }
1581
1582 int
cpu_clockspeed(int * freq)1583 cpu_clockspeed(int *freq)
1584 {
1585 *freq = clock_get_frequency(cpu_node, NULL) / 1000000;
1586 return 0;
1587 }
1588
1589 #ifdef MULTIPROCESSOR
1590
1591 void cpu_boot_secondary(struct cpu_info *ci);
1592 void cpu_hatch_secondary(void);
1593 void cpu_hatch_secondary_spin(void);
1594
1595 void cpu_suspend_cycle(void);
1596
1597 void
cpu_boot_secondary_processors(void)1598 cpu_boot_secondary_processors(void)
1599 {
1600 struct cpu_info *ci;
1601 CPU_INFO_ITERATOR cii;
1602
1603 CPU_INFO_FOREACH(cii, ci) {
1604 if ((ci->ci_flags & CPUF_AP) == 0)
1605 continue;
1606 if (ci->ci_flags & CPUF_PRIMARY)
1607 continue;
1608
1609 ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
1610 cpu_boot_secondary(ci);
1611 }
1612 }
1613
1614 void
cpu_start_spin_table(struct cpu_info * ci,uint64_t start,uint64_t data)1615 cpu_start_spin_table(struct cpu_info *ci, uint64_t start, uint64_t data)
1616 {
1617 extern paddr_t cpu_hatch_ci;
1618
1619 pmap_extract(pmap_kernel(), (vaddr_t)ci, &cpu_hatch_ci);
1620 cpu_dcache_wb_range((vaddr_t)&cpu_hatch_ci, sizeof(paddr_t));
1621
1622 /* this reuses the zero page for the core */
1623 vaddr_t start_pg = zero_page + (PAGE_SIZE * ci->ci_cpuid);
1624 paddr_t pa = trunc_page(data);
1625 uint64_t offset = data - pa;
1626 uint64_t *startvec = (uint64_t *)(start_pg + offset);
1627
1628 pmap_kenter_cache(start_pg, pa, PROT_READ|PROT_WRITE, PMAP_CACHE_CI);
1629
1630 *startvec = start;
1631 __asm volatile("dsb sy; sev" ::: "memory");
1632
1633 pmap_kremove(start_pg, PAGE_SIZE);
1634 }
1635
1636 int
cpu_start_secondary(struct cpu_info * ci,int method,uint64_t data)1637 cpu_start_secondary(struct cpu_info *ci, int method, uint64_t data)
1638 {
1639 vaddr_t start_va;
1640 paddr_t ci_pa, start_pa;
1641 uint64_t ttbr1;
1642 int32_t status;
1643
1644 __asm("mrs %x0, ttbr1_el1": "=r"(ttbr1));
1645 ci->ci_ttbr1 = ttbr1;
1646 cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci));
1647
1648 switch (method) {
1649 #if NPSCI > 0
1650 case 1:
1651 /* psci */
1652 start_va = (vaddr_t)cpu_hatch_secondary;
1653 pmap_extract(pmap_kernel(), start_va, &start_pa);
1654 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1655 status = psci_cpu_on(ci->ci_mpidr, start_pa, ci_pa);
1656 return (status == PSCI_SUCCESS);
1657 #endif
1658 case 2:
1659 /* spin-table */
1660 start_va = (vaddr_t)cpu_hatch_secondary_spin;
1661 pmap_extract(pmap_kernel(), start_va, &start_pa);
1662 cpu_start_spin_table(ci, start_pa, data);
1663 return 1;
1664 }
1665
1666 return 0;
1667 }
1668
1669 void
cpu_boot_secondary(struct cpu_info * ci)1670 cpu_boot_secondary(struct cpu_info *ci)
1671 {
1672 atomic_setbits_int(&ci->ci_flags, CPUF_GO);
1673 __asm volatile("dsb sy; sev" ::: "memory");
1674
1675 /*
1676 * Send an interrupt as well to make sure the CPU wakes up
1677 * regardless of whether it is in a WFE or a WFI loop.
1678 */
1679 arm_send_ipi(ci, ARM_IPI_NOP);
1680
1681 while ((ci->ci_flags & CPUF_RUNNING) == 0)
1682 __asm volatile("wfe");
1683 }
1684
1685 void
cpu_init_secondary(struct cpu_info * ci)1686 cpu_init_secondary(struct cpu_info *ci)
1687 {
1688 struct proc *p;
1689 struct pcb *pcb;
1690 struct trapframe *tf;
1691 struct switchframe *sf;
1692 int s;
1693
1694 ci->ci_flags |= CPUF_PRESENT;
1695 __asm volatile("dsb sy" ::: "memory");
1696
1697 if ((ci->ci_flags & CPUF_IDENTIFIED) == 0) {
1698 while ((ci->ci_flags & CPUF_IDENTIFY) == 0)
1699 __asm volatile("wfe");
1700
1701 cpu_identify(ci);
1702 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFIED);
1703 __asm volatile("dsb sy" ::: "memory");
1704 }
1705
1706 while ((ci->ci_flags & CPUF_GO) == 0)
1707 __asm volatile("wfe");
1708
1709 cpu_init();
1710
1711 /*
1712 * Start from a clean slate regardless of whether this is the
1713 * initial power up or a wakeup of a suspended CPU.
1714 */
1715
1716 ci->ci_curproc = NULL;
1717 ci->ci_curpcb = NULL;
1718 ci->ci_curpm = NULL;
1719 ci->ci_cpl = IPL_NONE;
1720 ci->ci_ipending = 0;
1721 ci->ci_idepth = 0;
1722
1723 #ifdef DIAGNOSTIC
1724 ci->ci_mutex_level = 0;
1725 #endif
1726
1727 /*
1728 * Re-create the switchframe for this CPUs idle process.
1729 */
1730
1731 p = ci->ci_schedstate.spc_idleproc;
1732 pcb = &p->p_addr->u_pcb;
1733
1734 tf = (struct trapframe *)((u_long)p->p_addr
1735 + USPACE
1736 - sizeof(struct trapframe)
1737 - 0x10);
1738
1739 tf = (struct trapframe *)STACKALIGN(tf);
1740 pcb->pcb_tf = tf;
1741
1742 sf = (struct switchframe *)tf - 1;
1743 sf->sf_x19 = (uint64_t)sched_idle;
1744 sf->sf_x20 = (uint64_t)ci;
1745 sf->sf_lr = (uint64_t)proc_trampoline;
1746 pcb->pcb_sp = (uint64_t)sf;
1747
1748 s = splhigh();
1749 arm_intr_cpu_enable();
1750 cpu_startclock();
1751
1752 atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
1753 __asm volatile("dsb sy; sev" ::: "memory");
1754
1755 spllower(IPL_NONE);
1756
1757 sched_toidle();
1758 }
1759
1760 void
cpu_halt(void)1761 cpu_halt(void)
1762 {
1763 struct cpu_info *ci = curcpu();
1764 vaddr_t start_va;
1765 paddr_t ci_pa, start_pa;
1766 int count = 0;
1767 u_long psw;
1768 int32_t status;
1769
1770 KERNEL_ASSERT_UNLOCKED();
1771 SCHED_ASSERT_UNLOCKED();
1772
1773 start_va = (vaddr_t)cpu_hatch_secondary;
1774 pmap_extract(pmap_kernel(), start_va, &start_pa);
1775 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1776
1777 psw = intr_disable();
1778
1779 atomic_clearbits_int(&ci->ci_flags,
1780 CPUF_RUNNING | CPUF_PRESENT | CPUF_GO);
1781
1782 #if NPSCI > 0
1783 if (psci_can_suspend())
1784 psci_cpu_off();
1785 #endif
1786
1787 /*
1788 * If we failed to turn ourselves off using PSCI, declare that
1789 * we're still present and spin in a low power state until
1790 * we're told to wake up again by the primary CPU.
1791 */
1792
1793 atomic_setbits_int(&ci->ci_flags, CPUF_PRESENT);
1794
1795 /* Mask clock interrupts. */
1796 WRITE_SPECIALREG(cntv_ctl_el0,
1797 READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK);
1798
1799 while ((ci->ci_flags & CPUF_GO) == 0) {
1800 #if NPSCI > 0
1801 if (ci->ci_psci_suspend_param) {
1802 status = psci_cpu_suspend(ci->ci_psci_suspend_param,
1803 start_pa, ci_pa);
1804 if (status != PSCI_SUCCESS)
1805 ci->ci_psci_suspend_param = 0;
1806 } else
1807 #endif
1808 cpu_suspend_cycle();
1809 count++;
1810 }
1811
1812 atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
1813 __asm volatile("dsb sy; sev" ::: "memory");
1814
1815 intr_restore(psw);
1816
1817 /* Unmask clock interrupts. */
1818 WRITE_SPECIALREG(cntv_ctl_el0,
1819 READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK);
1820 }
1821
1822 void
cpu_kick(struct cpu_info * ci)1823 cpu_kick(struct cpu_info *ci)
1824 {
1825 /* force cpu to enter kernel */
1826 if (ci != curcpu())
1827 arm_send_ipi(ci, ARM_IPI_NOP);
1828 }
1829
1830 void
cpu_unidle(struct cpu_info * ci)1831 cpu_unidle(struct cpu_info *ci)
1832 {
1833 /*
1834 * This could send IPI or SEV depending on if the other
1835 * processor is sleeping (WFI or WFE), in userland, or if the
1836 * cpu is in other possible wait states?
1837 */
1838 if (ci != curcpu())
1839 arm_send_ipi(ci, ARM_IPI_NOP);
1840 }
1841
1842 #endif
1843
1844 int cpu_suspended;
1845
1846 #ifdef SUSPEND
1847
1848 void cpu_hatch_primary(void);
1849
1850 void (*cpu_suspend_cycle_fcn)(void) = cpu_wfi;
1851 label_t cpu_suspend_jmpbuf;
1852
1853 void
cpu_suspend_cycle(void)1854 cpu_suspend_cycle(void)
1855 {
1856 cpu_suspend_cycle_fcn();
1857 }
1858
1859 void
cpu_init_primary(void)1860 cpu_init_primary(void)
1861 {
1862 cpu_init();
1863
1864 cpu_startclock();
1865
1866 longjmp(&cpu_suspend_jmpbuf);
1867 }
1868
1869 int
cpu_suspend_primary(void)1870 cpu_suspend_primary(void)
1871 {
1872 struct cpu_info *ci = curcpu();
1873 vaddr_t start_va;
1874 paddr_t ci_pa, start_pa;
1875 uint64_t ttbr1;
1876 int32_t status;
1877 int count = 0;
1878
1879 __asm("mrs %x0, ttbr1_el1": "=r"(ttbr1));
1880 ci->ci_ttbr1 = ttbr1;
1881 cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci));
1882
1883 start_va = (vaddr_t)cpu_hatch_primary;
1884 pmap_extract(pmap_kernel(), start_va, &start_pa);
1885 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1886
1887 #if NPSCI > 0
1888 if (psci_can_suspend()) {
1889 if (setjmp(&cpu_suspend_jmpbuf)) {
1890 /* XXX wait for debug output on Allwinner A64 */
1891 delay(200000);
1892 return 0;
1893 }
1894
1895 psci_system_suspend(start_pa, ci_pa);
1896
1897 return EOPNOTSUPP;
1898 }
1899 #endif
1900
1901 if (setjmp(&cpu_suspend_jmpbuf))
1902 goto resume;
1903
1904 /*
1905 * If PSCI doesn't support SYSTEM_SUSPEND, spin in a low power
1906 * state waiting for an interrupt that wakes us up again.
1907 */
1908
1909 /* Mask clock interrupts. */
1910 WRITE_SPECIALREG(cntv_ctl_el0,
1911 READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK);
1912
1913 /*
1914 * All non-wakeup interrupts should be masked at this point;
1915 * re-enable interrupts such that wakeup interrupts actually
1916 * wake us up. Set a flag such that drivers can tell we're
1917 * suspended and change their behaviour accordingly. They can
1918 * wake us up by clearing the flag.
1919 */
1920 cpu_suspended = 1;
1921 arm_intr_func.setipl(IPL_NONE);
1922 intr_enable();
1923
1924 while (cpu_suspended) {
1925 #if NPSCI > 0
1926 if (ci->ci_psci_suspend_param) {
1927 status = psci_cpu_suspend(ci->ci_psci_suspend_param,
1928 start_pa, ci_pa);
1929 if (status != PSCI_SUCCESS)
1930 ci->ci_psci_suspend_param = 0;
1931 } else
1932 #endif
1933 cpu_suspend_cycle();
1934 count++;
1935 }
1936
1937 resume:
1938 intr_disable();
1939 arm_intr_func.setipl(IPL_HIGH);
1940
1941 /* Unmask clock interrupts. */
1942 WRITE_SPECIALREG(cntv_ctl_el0,
1943 READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK);
1944
1945 return 0;
1946 }
1947
1948 #ifdef MULTIPROCESSOR
1949
1950 void
cpu_resume_secondary(struct cpu_info * ci)1951 cpu_resume_secondary(struct cpu_info *ci)
1952 {
1953 int timeout = 10000;
1954
1955 if (ci->ci_flags & CPUF_PRESENT)
1956 return;
1957
1958 cpu_start_secondary(ci, 1, 0);
1959 while ((ci->ci_flags & CPUF_PRESENT) == 0 && --timeout)
1960 delay(1000);
1961 if (timeout == 0) {
1962 printf("%s: failed to spin up\n",
1963 ci->ci_dev->dv_xname);
1964 ci->ci_flags = 0;
1965 }
1966 }
1967
1968 #endif
1969
1970 #endif
1971
1972 /*
1973 * Dynamic voltage and frequency scaling implementation.
1974 */
1975
1976 extern int perflevel;
1977
1978 struct opp {
1979 uint64_t opp_hz;
1980 uint32_t opp_microvolt;
1981 };
1982
1983 struct opp_table {
1984 LIST_ENTRY(opp_table) ot_list;
1985 uint32_t ot_phandle;
1986
1987 struct opp *ot_opp;
1988 u_int ot_nopp;
1989 uint64_t ot_opp_hz_min;
1990 uint64_t ot_opp_hz_max;
1991
1992 struct cpu_info *ot_master;
1993 };
1994
1995 LIST_HEAD(, opp_table) opp_tables = LIST_HEAD_INITIALIZER(opp_tables);
1996 struct task cpu_opp_task;
1997
1998 void cpu_opp_mountroot(struct device *);
1999 void cpu_opp_dotask(void *);
2000 void cpu_opp_setperf(int);
2001
2002 uint32_t cpu_opp_get_cooling_level(void *, uint32_t *);
2003 void cpu_opp_set_cooling_level(void *, uint32_t *, uint32_t);
2004
2005 void
cpu_opp_init(struct cpu_info * ci,uint32_t phandle)2006 cpu_opp_init(struct cpu_info *ci, uint32_t phandle)
2007 {
2008 struct opp_table *ot;
2009 struct cooling_device *cd;
2010 int count, node, child;
2011 uint32_t opp_hz, opp_microvolt;
2012 uint32_t values[3];
2013 int i, j, len;
2014
2015 LIST_FOREACH(ot, &opp_tables, ot_list) {
2016 if (ot->ot_phandle == phandle) {
2017 ci->ci_opp_table = ot;
2018 return;
2019 }
2020 }
2021
2022 node = OF_getnodebyphandle(phandle);
2023 if (node == 0)
2024 return;
2025
2026 if (!OF_is_compatible(node, "operating-points-v2"))
2027 return;
2028
2029 count = 0;
2030 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
2031 if (OF_getproplen(child, "turbo-mode") == 0)
2032 continue;
2033 count++;
2034 }
2035 if (count == 0)
2036 return;
2037
2038 ot = malloc(sizeof(struct opp_table), M_DEVBUF, M_ZERO | M_WAITOK);
2039 ot->ot_phandle = phandle;
2040 ot->ot_opp = mallocarray(count, sizeof(struct opp),
2041 M_DEVBUF, M_ZERO | M_WAITOK);
2042 ot->ot_nopp = count;
2043
2044 count = 0;
2045 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
2046 if (OF_getproplen(child, "turbo-mode") == 0)
2047 continue;
2048 opp_hz = OF_getpropint64(child, "opp-hz", 0);
2049 len = OF_getpropintarray(child, "opp-microvolt",
2050 values, sizeof(values));
2051 opp_microvolt = 0;
2052 if (len == sizeof(uint32_t) || len == 3 * sizeof(uint32_t))
2053 opp_microvolt = values[0];
2054
2055 /* Insert into the array, keeping things sorted. */
2056 for (i = 0; i < count; i++) {
2057 if (opp_hz < ot->ot_opp[i].opp_hz)
2058 break;
2059 }
2060 for (j = count; j > i; j--)
2061 ot->ot_opp[j] = ot->ot_opp[j - 1];
2062 ot->ot_opp[i].opp_hz = opp_hz;
2063 ot->ot_opp[i].opp_microvolt = opp_microvolt;
2064 count++;
2065 }
2066
2067 ot->ot_opp_hz_min = ot->ot_opp[0].opp_hz;
2068 ot->ot_opp_hz_max = ot->ot_opp[count - 1].opp_hz;
2069
2070 if (OF_getproplen(node, "opp-shared") == 0)
2071 ot->ot_master = ci;
2072
2073 LIST_INSERT_HEAD(&opp_tables, ot, ot_list);
2074
2075 ci->ci_opp_table = ot;
2076 ci->ci_opp_max = ot->ot_nopp - 1;
2077 ci->ci_cpu_supply = OF_getpropint(ci->ci_node, "cpu-supply", 0);
2078
2079 cd = malloc(sizeof(struct cooling_device), M_DEVBUF, M_ZERO | M_WAITOK);
2080 cd->cd_node = ci->ci_node;
2081 cd->cd_cookie = ci;
2082 cd->cd_get_level = cpu_opp_get_cooling_level;
2083 cd->cd_set_level = cpu_opp_set_cooling_level;
2084 cooling_device_register(cd);
2085
2086 /*
2087 * Do additional checks at mountroot when all the clocks and
2088 * regulators are available.
2089 */
2090 config_mountroot(ci->ci_dev, cpu_opp_mountroot);
2091 }
2092
2093 void
cpu_opp_mountroot(struct device * self)2094 cpu_opp_mountroot(struct device *self)
2095 {
2096 struct cpu_info *ci;
2097 CPU_INFO_ITERATOR cii;
2098 int count = 0;
2099 int level = 0;
2100
2101 if (cpu_setperf)
2102 return;
2103
2104 CPU_INFO_FOREACH(cii, ci) {
2105 struct opp_table *ot = ci->ci_opp_table;
2106 uint64_t curr_hz;
2107 uint32_t curr_microvolt;
2108 int error;
2109
2110 if (ot == NULL)
2111 continue;
2112
2113 #if NKSTAT > 0
2114 cpu_opp_kstat_attach(ci);
2115 #endif
2116
2117 /* Skip if this table is shared and we're not the master. */
2118 if (ot->ot_master && ot->ot_master != ci)
2119 continue;
2120
2121 /* PWM regulators may need to be explicitly enabled. */
2122 regulator_enable(ci->ci_cpu_supply);
2123
2124 curr_hz = clock_get_frequency(ci->ci_node, NULL);
2125 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
2126
2127 /* Disable if clock isn't implemented. */
2128 error = ENODEV;
2129 if (curr_hz != 0)
2130 error = clock_set_frequency(ci->ci_node, NULL, curr_hz);
2131 if (error) {
2132 ci->ci_opp_table = NULL;
2133 printf("%s: clock not implemented\n",
2134 ci->ci_dev->dv_xname);
2135 continue;
2136 }
2137
2138 /* Disable if regulator isn't implemented. */
2139 error = ci->ci_cpu_supply ? ENODEV : 0;
2140 if (ci->ci_cpu_supply && curr_microvolt != 0)
2141 error = regulator_set_voltage(ci->ci_cpu_supply,
2142 curr_microvolt);
2143 if (error) {
2144 ci->ci_opp_table = NULL;
2145 printf("%s: regulator not implemented\n",
2146 ci->ci_dev->dv_xname);
2147 continue;
2148 }
2149
2150 /*
2151 * Initialize performance level based on the current
2152 * speed of the first CPU that supports DVFS.
2153 */
2154 if (level == 0) {
2155 uint64_t min, max;
2156 uint64_t level_hz;
2157
2158 min = ot->ot_opp_hz_min;
2159 max = ot->ot_opp_hz_max;
2160 level_hz = clock_get_frequency(ci->ci_node, NULL);
2161 if (level_hz < min)
2162 level_hz = min;
2163 if (level_hz > max)
2164 level_hz = max;
2165 level = howmany(100 * (level_hz - min), (max - min));
2166 }
2167
2168 count++;
2169 }
2170
2171 if (count > 0) {
2172 task_set(&cpu_opp_task, cpu_opp_dotask, NULL);
2173 cpu_setperf = cpu_opp_setperf;
2174
2175 perflevel = (level > 0) ? level : 0;
2176 cpu_setperf(perflevel);
2177 }
2178 }
2179
2180 void
cpu_opp_dotask(void * arg)2181 cpu_opp_dotask(void *arg)
2182 {
2183 struct cpu_info *ci;
2184 CPU_INFO_ITERATOR cii;
2185
2186 CPU_INFO_FOREACH(cii, ci) {
2187 struct opp_table *ot = ci->ci_opp_table;
2188 uint64_t curr_hz, opp_hz;
2189 uint32_t curr_microvolt, opp_microvolt;
2190 int opp_idx;
2191 int error = 0;
2192
2193 if (ot == NULL)
2194 continue;
2195
2196 /* Skip if this table is shared and we're not the master. */
2197 if (ot->ot_master && ot->ot_master != ci)
2198 continue;
2199
2200 opp_idx = MIN(ci->ci_opp_idx, ci->ci_opp_max);
2201 opp_hz = ot->ot_opp[opp_idx].opp_hz;
2202 opp_microvolt = ot->ot_opp[opp_idx].opp_microvolt;
2203
2204 curr_hz = clock_get_frequency(ci->ci_node, NULL);
2205 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
2206
2207 if (error == 0 && opp_hz < curr_hz)
2208 error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
2209 if (error == 0 && ci->ci_cpu_supply &&
2210 opp_microvolt != 0 && opp_microvolt != curr_microvolt) {
2211 error = regulator_set_voltage(ci->ci_cpu_supply,
2212 opp_microvolt);
2213 }
2214 if (error == 0 && opp_hz > curr_hz)
2215 error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
2216
2217 if (error)
2218 printf("%s: DVFS failed\n", ci->ci_dev->dv_xname);
2219 }
2220 }
2221
2222 void
cpu_opp_setperf(int level)2223 cpu_opp_setperf(int level)
2224 {
2225 struct cpu_info *ci;
2226 CPU_INFO_ITERATOR cii;
2227
2228 CPU_INFO_FOREACH(cii, ci) {
2229 struct opp_table *ot = ci->ci_opp_table;
2230 uint64_t min, max;
2231 uint64_t level_hz, opp_hz;
2232 int opp_idx = -1;
2233 int i;
2234
2235 if (ot == NULL)
2236 continue;
2237
2238 /* Skip if this table is shared and we're not the master. */
2239 if (ot->ot_master && ot->ot_master != ci)
2240 continue;
2241
2242 min = ot->ot_opp_hz_min;
2243 max = ot->ot_opp_hz_max;
2244 level_hz = min + (level * (max - min)) / 100;
2245 opp_hz = min;
2246 for (i = 0; i < ot->ot_nopp; i++) {
2247 if (ot->ot_opp[i].opp_hz <= level_hz &&
2248 ot->ot_opp[i].opp_hz >= opp_hz)
2249 opp_hz = ot->ot_opp[i].opp_hz;
2250 }
2251
2252 /* Find index of selected operating point. */
2253 for (i = 0; i < ot->ot_nopp; i++) {
2254 if (ot->ot_opp[i].opp_hz == opp_hz) {
2255 opp_idx = i;
2256 break;
2257 }
2258 }
2259 KASSERT(opp_idx >= 0);
2260
2261 ci->ci_opp_idx = opp_idx;
2262 }
2263
2264 /*
2265 * Update the hardware from a task since setting the
2266 * regulators might need process context.
2267 */
2268 task_add(systq, &cpu_opp_task);
2269 }
2270
2271 uint32_t
cpu_opp_get_cooling_level(void * cookie,uint32_t * cells)2272 cpu_opp_get_cooling_level(void *cookie, uint32_t *cells)
2273 {
2274 struct cpu_info *ci = cookie;
2275 struct opp_table *ot = ci->ci_opp_table;
2276
2277 return ot->ot_nopp - ci->ci_opp_max - 1;
2278 }
2279
2280 void
cpu_opp_set_cooling_level(void * cookie,uint32_t * cells,uint32_t level)2281 cpu_opp_set_cooling_level(void *cookie, uint32_t *cells, uint32_t level)
2282 {
2283 struct cpu_info *ci = cookie;
2284 struct opp_table *ot = ci->ci_opp_table;
2285 int opp_max;
2286
2287 if (level > (ot->ot_nopp - 1))
2288 level = ot->ot_nopp - 1;
2289
2290 opp_max = (ot->ot_nopp - level - 1);
2291 if (ci->ci_opp_max != opp_max) {
2292 ci->ci_opp_max = opp_max;
2293 task_add(systq, &cpu_opp_task);
2294 }
2295 }
2296
2297
2298 void
cpu_psci_init(struct cpu_info * ci)2299 cpu_psci_init(struct cpu_info *ci)
2300 {
2301 uint32_t *domains;
2302 uint32_t *domain;
2303 uint32_t *states;
2304 uint32_t ncells;
2305 uint32_t cluster;
2306 int idx, len, node;
2307
2308 /*
2309 * Find the shallowest (for now) idle state for this CPU.
2310 * This should be the first one that is listed. We'll use it
2311 * in the idle loop.
2312 */
2313
2314 len = OF_getproplen(ci->ci_node, "cpu-idle-states");
2315 if (len < (int)sizeof(uint32_t))
2316 return;
2317
2318 states = malloc(len, M_TEMP, M_WAITOK);
2319 OF_getpropintarray(ci->ci_node, "cpu-idle-states", states, len);
2320 node = OF_getnodebyphandle(states[0]);
2321 free(states, M_TEMP, len);
2322 if (node) {
2323 uint32_t entry, exit, residency, param;
2324 int32_t features;
2325
2326 param = OF_getpropint(node, "arm,psci-suspend-param", 0);
2327 entry = OF_getpropint(node, "entry-latency-us", 0);
2328 exit = OF_getpropint(node, "exit-latency-us", 0);
2329 residency = OF_getpropint(node, "min-residency-us", 0);
2330 ci->ci_psci_idle_latency += entry + exit + 2 * residency;
2331
2332 /* Skip states that stop the local timer. */
2333 if (OF_getpropbool(node, "local-timer-stop"))
2334 ci->ci_psci_idle_param = 0;
2335
2336 /* Skip powerdown states. */
2337 features = psci_features(CPU_SUSPEND);
2338 if (features == PSCI_NOT_SUPPORTED ||
2339 (features & PSCI_FEATURE_POWER_STATE_EXT) == 0) {
2340 if (param & PSCI_POWER_STATE_POWERDOWN)
2341 param = 0;
2342 } else {
2343 if (param & PSCI_POWER_STATE_EXT_POWERDOWN)
2344 param = 0;
2345 }
2346
2347 if (param) {
2348 ci->ci_psci_idle_param = param;
2349 cpu_idle_cycle_fcn = cpu_psci_idle_cycle;
2350 }
2351 }
2352
2353 /*
2354 * Hunt for the deepest idle state for this CPU. This is
2355 * fairly complicated as it requires traversing quite a few
2356 * nodes in the device tree. The first step is to look up the
2357 * "psci" power domain for this CPU.
2358 */
2359
2360 idx = OF_getindex(ci->ci_node, "psci", "power-domain-names");
2361 if (idx < 0)
2362 return;
2363
2364 len = OF_getproplen(ci->ci_node, "power-domains");
2365 if (len <= 0)
2366 return;
2367
2368 domains = malloc(len, M_TEMP, M_WAITOK);
2369 OF_getpropintarray(ci->ci_node, "power-domains", domains, len);
2370
2371 domain = domains;
2372 while (domain && domain < domains + (len / sizeof(uint32_t))) {
2373 if (idx == 0)
2374 break;
2375
2376 node = OF_getnodebyphandle(domain[0]);
2377 if (node == 0)
2378 break;
2379
2380 ncells = OF_getpropint(node, "#power-domain-cells", 0);
2381 domain = domain + ncells + 1;
2382 idx--;
2383 }
2384
2385 node = idx == 0 ? OF_getnodebyphandle(domain[0]) : 0;
2386 free(domains, M_TEMP, len);
2387 if (node == 0)
2388 return;
2389
2390 /*
2391 * We found the "psci" power domain. If this power domain has
2392 * a parent power domain, stash its phandle away for later.
2393 */
2394
2395 cluster = OF_getpropint(node, "power-domains", 0);
2396
2397 /*
2398 * Get the deepest idle state for the CPU; this should be the
2399 * last one that is listed.
2400 */
2401
2402 len = OF_getproplen(node, "domain-idle-states");
2403 if (len < (int)sizeof(uint32_t))
2404 return;
2405
2406 states = malloc(len, M_TEMP, M_WAITOK);
2407 OF_getpropintarray(node, "domain-idle-states", states, len);
2408
2409 node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]);
2410 free(states, M_TEMP, len);
2411 if (node == 0)
2412 return;
2413
2414 ci->ci_psci_suspend_param =
2415 OF_getpropint(node, "arm,psci-suspend-param", 0);
2416
2417 /*
2418 * Qualcomm Snapdragon always seem to operate in OS Initiated
2419 * mode. This means that the last CPU to suspend can pick the
2420 * idle state that powers off the entire cluster. In our case
2421 * that will always be the primary CPU.
2422 */
2423
2424 #ifdef MULTIPROCESSOR
2425 if (ci->ci_flags & CPUF_AP)
2426 return;
2427 #endif
2428
2429 node = OF_getnodebyphandle(cluster);
2430 if (node == 0)
2431 return;
2432
2433 /*
2434 * Get the deepest idle state for the cluster; this should be
2435 * the last one that is listed.
2436 */
2437
2438 states = malloc(len, M_TEMP, M_WAITOK);
2439 OF_getpropintarray(node, "domain-idle-states", states, len);
2440
2441 node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]);
2442 free(states, M_TEMP, len);
2443 if (node == 0)
2444 return;
2445
2446 ci->ci_psci_suspend_param =
2447 OF_getpropint(node, "arm,psci-suspend-param", 0);
2448 }
2449
2450 void
cpu_psci_idle_cycle(void)2451 cpu_psci_idle_cycle(void)
2452 {
2453 struct cpu_info *ci = curcpu();
2454 struct timeval start, stop;
2455 u_long itime;
2456
2457 microuptime(&start);
2458
2459 if (ci->ci_prev_sleep > ci->ci_psci_idle_latency)
2460 psci_cpu_suspend(ci->ci_psci_idle_param, 0, 0);
2461 else
2462 cpu_wfi();
2463
2464 microuptime(&stop);
2465 timersub(&stop, &start, &stop);
2466 itime = stop.tv_sec * 1000000 + stop.tv_usec;
2467
2468 ci->ci_last_itime = itime;
2469 itime >>= 1;
2470 ci->ci_prev_sleep = (ci->ci_prev_sleep + (ci->ci_prev_sleep >> 1)
2471 + itime) >> 1;
2472 }
2473
2474 #if NKSTAT > 0
2475
2476 struct cpu_kstats {
2477 struct kstat_kv ck_impl;
2478 struct kstat_kv ck_part;
2479 struct kstat_kv ck_rev;
2480 };
2481
2482 void
cpu_kstat_attach(struct cpu_info * ci)2483 cpu_kstat_attach(struct cpu_info *ci)
2484 {
2485 struct kstat *ks;
2486 struct cpu_kstats *ck;
2487 uint64_t impl, part;
2488 const char *impl_name = NULL, *part_name = NULL;
2489 const struct cpu_cores *coreselecter = cpu_cores_none;
2490 int i;
2491
2492 ks = kstat_create(ci->ci_dev->dv_xname, 0, "mach", 0, KSTAT_T_KV, 0);
2493 if (ks == NULL) {
2494 printf("%s: unable to create cpu kstats\n",
2495 ci->ci_dev->dv_xname);
2496 return;
2497 }
2498
2499 ck = malloc(sizeof(*ck), M_DEVBUF, M_WAITOK);
2500
2501 impl = CPU_IMPL(ci->ci_midr);
2502 part = CPU_PART(ci->ci_midr);
2503
2504 for (i = 0; cpu_implementers[i].name; i++) {
2505 if (impl == cpu_implementers[i].id) {
2506 impl_name = cpu_implementers[i].name;
2507 coreselecter = cpu_implementers[i].corelist;
2508 break;
2509 }
2510 }
2511
2512 if (impl_name) {
2513 kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_ISTR);
2514 strlcpy(kstat_kv_istr(&ck->ck_impl), impl_name,
2515 sizeof(kstat_kv_istr(&ck->ck_impl)));
2516 } else
2517 kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_NULL);
2518
2519 for (i = 0; coreselecter[i].name; i++) {
2520 if (part == coreselecter[i].id) {
2521 part_name = coreselecter[i].name;
2522 break;
2523 }
2524 }
2525
2526 if (part_name) {
2527 kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_ISTR);
2528 strlcpy(kstat_kv_istr(&ck->ck_part), part_name,
2529 sizeof(kstat_kv_istr(&ck->ck_part)));
2530 } else
2531 kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_NULL);
2532
2533 kstat_kv_init(&ck->ck_rev, "rev", KSTAT_KV_T_ISTR);
2534 snprintf(kstat_kv_istr(&ck->ck_rev), sizeof(kstat_kv_istr(&ck->ck_rev)),
2535 "r%llup%llu", CPU_VAR(ci->ci_midr), CPU_REV(ci->ci_midr));
2536
2537 ks->ks_softc = ci;
2538 ks->ks_data = ck;
2539 ks->ks_datalen = sizeof(*ck);
2540 ks->ks_read = kstat_read_nop;
2541
2542 kstat_install(ks);
2543
2544 /* XXX should we have a ci->ci_kstat = ks? */
2545 }
2546
2547 struct cpu_opp_kstats {
2548 struct kstat_kv coppk_freq;
2549 struct kstat_kv coppk_supply_v;
2550 };
2551
2552 int
cpu_opp_kstat_read(struct kstat * ks)2553 cpu_opp_kstat_read(struct kstat *ks)
2554 {
2555 struct cpu_info *ci = ks->ks_softc;
2556 struct cpu_opp_kstats *coppk = ks->ks_data;
2557
2558 struct opp_table *ot = ci->ci_opp_table;
2559 struct cpu_info *oci;
2560 struct timespec now, diff;
2561
2562 /* rate limit */
2563 getnanouptime(&now);
2564 timespecsub(&now, &ks->ks_updated, &diff);
2565 if (diff.tv_sec < 1)
2566 return (0);
2567
2568 if (ot == NULL)
2569 return (0);
2570
2571 oci = ot->ot_master;
2572 if (oci == NULL)
2573 oci = ci;
2574
2575 kstat_kv_freq(&coppk->coppk_freq) =
2576 clock_get_frequency(oci->ci_node, NULL);
2577
2578 if (oci->ci_cpu_supply) {
2579 kstat_kv_volts(&coppk->coppk_supply_v) =
2580 regulator_get_voltage(oci->ci_cpu_supply);
2581 }
2582
2583 ks->ks_updated = now;
2584
2585 return (0);
2586 }
2587
2588 void
cpu_opp_kstat_attach(struct cpu_info * ci)2589 cpu_opp_kstat_attach(struct cpu_info *ci)
2590 {
2591 struct kstat *ks;
2592 struct cpu_opp_kstats *coppk;
2593 struct opp_table *ot = ci->ci_opp_table;
2594 struct cpu_info *oci = ot->ot_master;
2595
2596 if (oci == NULL)
2597 oci = ci;
2598
2599 ks = kstat_create(ci->ci_dev->dv_xname, 0, "dt-opp", 0,
2600 KSTAT_T_KV, 0);
2601 if (ks == NULL) {
2602 printf("%s: unable to create cpu dt-opp kstats\n",
2603 ci->ci_dev->dv_xname);
2604 return;
2605 }
2606
2607 coppk = malloc(sizeof(*coppk), M_DEVBUF, M_WAITOK);
2608
2609 kstat_kv_init(&coppk->coppk_freq, "freq", KSTAT_KV_T_FREQ);
2610 kstat_kv_init(&coppk->coppk_supply_v, "supply",
2611 oci->ci_cpu_supply ? KSTAT_KV_T_VOLTS_DC : KSTAT_KV_T_NULL);
2612
2613 ks->ks_softc = oci;
2614 ks->ks_data = coppk;
2615 ks->ks_datalen = sizeof(*coppk);
2616 ks->ks_read = cpu_opp_kstat_read;
2617
2618 kstat_install(ks);
2619
2620 /* XXX should we have a ci->ci_opp_kstat = ks? */
2621 }
2622
2623 #endif /* NKSTAT > 0 */
2624