1 /* $NetBSD: arm32_machdep.c,v 1.147 2024/03/05 14:15:29 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 1994-1998 Mark Brinicombe.
5 * Copyright (c) 1994 Brini.
6 * All rights reserved.
7 *
8 * This code is derived from software written for Brini by Mark Brinicombe
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by Mark Brinicombe
21 * for the NetBSD Project.
22 * 4. The name of the company nor the name of the author may be used to
23 * endorse or promote products derived from this software without specific
24 * prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
27 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
28 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
29 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
30 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
31 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
32 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * Machine dependent functions for kernel setup
39 *
40 * Created : 17/09/94
41 * Updated : 18/04/01 updated for new wscons
42 */
43
44 #include <sys/cdefs.h>
45 __KERNEL_RCSID(0, "$NetBSD: arm32_machdep.c,v 1.147 2024/03/05 14:15:29 thorpej Exp $");
46
47 #include "opt_arm_debug.h"
48 #include "opt_arm_start.h"
49 #include "opt_fdt.h"
50 #include "opt_modular.h"
51 #include "opt_md.h"
52 #include "opt_multiprocessor.h"
53
54 #include <sys/param.h>
55
56 #include <sys/atomic.h>
57 #include <sys/buf.h>
58 #include <sys/cpu.h>
59 #include <sys/device.h>
60 #include <sys/intr.h>
61 #include <sys/ipi.h>
62 #include <sys/kauth.h>
63 #include <sys/kernel.h>
64 #include <sys/mbuf.h>
65 #include <sys/module.h>
66 #include <sys/mount.h>
67 #include <sys/msgbuf.h>
68 #include <sys/proc.h>
69 #include <sys/reboot.h>
70 #include <sys/sysctl.h>
71 #include <sys/systm.h>
72 #include <sys/xcall.h>
73
74 #include <uvm/uvm_extern.h>
75
76 #include <dev/cons.h>
77 #include <dev/mm.h>
78
79 #include <arm/locore.h>
80
81 #include <arm/cpu_topology.h>
82 #include <arm/arm32/machdep.h>
83
84 #include <machine/bootconfig.h>
85 #include <machine/pcb.h>
86
87 #if defined(FDT)
88 #include <dev/fdt/fdtvar.h>
89
90 #include <arm/fdt/arm_fdtvar.h>
91 #include <arch/evbarm/fdt/platform.h>
92 #endif
93
94 #ifdef VERBOSE_INIT_ARM
95 #define VPRINTF(...) printf(__VA_ARGS__)
96 #ifdef __HAVE_GENERIC_START
97 void generic_prints(const char *);
98 void generic_printx(int);
99 #define VPRINTS(s) generic_prints(s)
100 #define VPRINTX(x) generic_printx(x)
101 #else
102 #define VPRINTS(s) __nothing
103 #define VPRINTX(x) __nothing
104 #endif
105 #else
106 #define VPRINTF(...) __nothing
107 #define VPRINTS(s) __nothing
108 #define VPRINTX(x) __nothing
109 #endif
110
111 void (*cpu_reset_address)(void); /* Used by locore */
112 paddr_t cpu_reset_address_paddr; /* Used by locore */
113
114 struct vm_map *phys_map = NULL;
115
116 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
117 extern size_t md_root_size; /* Memory disc size */
118 #endif /* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
119
120 pv_addr_t kernelstack;
121 pv_addr_t abtstack;
122 pv_addr_t fiqstack;
123 pv_addr_t irqstack;
124 pv_addr_t undstack;
125 pv_addr_t idlestack;
126
127 void * msgbufaddr;
128 extern paddr_t msgbufphys;
129
130 int kernel_debug = 0;
131 int cpu_printfataltraps = 0;
132 int cpu_fpu_present;
133 int cpu_hwdiv_present;
134 int cpu_neon_present;
135 int cpu_simd_present;
136 int cpu_simdex_present;
137 int cpu_umull_present;
138 int cpu_synchprim_present;
139 int cpu_unaligned_sigbus;
140 const char *cpu_arch = "";
141
142 int cpu_instruction_set_attributes[6];
143 int cpu_memory_model_features[4];
144 int cpu_processor_features[2];
145 int cpu_media_and_vfp_features[2];
146
147 /* exported variable to be filled in by the bootloaders */
148 char *booted_kernel;
149
150 /* Prototypes */
151
152 void data_abort_handler(trapframe_t *frame);
153 void prefetch_abort_handler(trapframe_t *frame);
154 extern void configure(void);
155
156 /*
157 * arm32_vector_init:
158 *
159 * Initialize the vector page, and select whether or not to
160 * relocate the vectors.
161 *
162 * NOTE: We expect the vector page to be mapped at its expected
163 * destination.
164 */
165 void
arm32_vector_init(vaddr_t va,int which)166 arm32_vector_init(vaddr_t va, int which)
167 {
168 #if defined(CPU_ARMV7) || defined(CPU_ARM11) || defined(ARM_HAS_VBAR)
169 /*
170 * If this processor has the security extension, don't bother
171 * to move/map the vector page. Simply point VBAR to the copy
172 * that exists in the .text segment.
173 */
174 #ifndef ARM_HAS_VBAR
175 if (va == ARM_VECTORS_LOW
176 && (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0) {
177 #endif
178 extern const uint32_t page0rel[];
179 vector_page = (vaddr_t)page0rel;
180 KASSERT((vector_page & 0x1f) == 0);
181 armreg_vbar_write(vector_page);
182 VPRINTF(" vbar=%p", page0rel);
183 cpu_control(CPU_CONTROL_VECRELOC, 0);
184 return;
185 #ifndef ARM_HAS_VBAR
186 }
187 #endif
188 #endif
189 #ifndef ARM_HAS_VBAR
190 if (CPU_IS_PRIMARY(curcpu())) {
191 extern unsigned int page0[], page0_data[];
192 unsigned int *vectors = (int *) va;
193 unsigned int *vectors_data = vectors + (page0_data - page0);
194 int vec;
195
196 /*
197 * Loop through the vectors we're taking over, and copy the
198 * vector's insn and data word.
199 */
200 for (vec = 0; vec < ARM_NVEC; vec++) {
201 if ((which & (1 << vec)) == 0) {
202 /* Don't want to take over this vector. */
203 continue;
204 }
205 vectors[vec] = page0[vec];
206 vectors_data[vec] = page0_data[vec];
207 }
208
209 /* Now sync the vectors. */
210 cpu_icache_sync_range(va, (ARM_NVEC * 2) * sizeof(u_int));
211
212 vector_page = va;
213 }
214
215 if (va == ARM_VECTORS_HIGH) {
216 /*
217 * Assume the MD caller knows what it's doing here, and
218 * really does want the vector page relocated.
219 *
220 * Note: This has to be done here (and not just in
221 * cpu_setup()) because the vector page needs to be
222 * accessible *before* cpu_startup() is called.
223 * Think ddb(9) ...
224 *
225 * NOTE: If the CPU control register is not readable,
226 * this will totally fail! We'll just assume that
227 * any system that has high vector support has a
228 * readable CPU control register, for now. If we
229 * ever encounter one that does not, we'll have to
230 * rethink this.
231 */
232 cpu_control(CPU_CONTROL_VECRELOC, CPU_CONTROL_VECRELOC);
233 }
234 #endif
235 }
236
237 /*
238 * Debug function just to park the CPU
239 */
240
241 void
halt(void)242 halt(void)
243 {
244 while (1)
245 cpu_sleep(0);
246 }
247
248
249 /* Sync the discs, unmount the filesystems, and adjust the todr */
250
251 void
bootsync(void)252 bootsync(void)
253 {
254 static bool bootsyncdone = false;
255
256 if (bootsyncdone) return;
257
258 bootsyncdone = true;
259
260 /* Make sure we can still manage to do things */
261 if (GetCPSR() & I32_bit) {
262 /*
263 * If we get here then boot has been called without RB_NOSYNC
264 * and interrupts were disabled. This means the boot() call
265 * did not come from a user process e.g. shutdown, but must
266 * have come from somewhere in the kernel.
267 */
268 IRQenable;
269 printf("Warning IRQ's disabled during boot()\n");
270 }
271
272 vfs_shutdown();
273 }
274
275 /*
276 * void cpu_startup(void)
277 *
278 * Machine dependent startup code.
279 *
280 */
281 void
cpu_startup(void)282 cpu_startup(void)
283 {
284 vaddr_t minaddr;
285 vaddr_t maxaddr;
286
287 #ifndef __HAVE_GENERIC_START
288 /* Set the CPU control register */
289 cpu_setup(boot_args);
290 #endif
291
292 #ifndef ARM_HAS_VBAR
293 /* Lock down zero page */
294 vector_page_setprot(VM_PROT_READ);
295 #endif
296
297 /*
298 * Give pmap a chance to set up a few more things now the vm
299 * is initialised
300 */
301 pmap_postinit();
302
303 #ifdef FDT
304 const struct fdt_platform * const plat = fdt_platform_find();
305 if (plat->fp_startup != NULL)
306 plat->fp_startup();
307 #endif
308
309 /*
310 * Initialize error message buffer (at end of core).
311 */
312
313 /* msgbufphys was setup during the secondary boot strap */
314 if (!pmap_extract(pmap_kernel(), (vaddr_t)msgbufaddr, NULL)) {
315 for (u_int loop = 0; loop < btoc(MSGBUFSIZE); ++loop) {
316 pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
317 msgbufphys + loop * PAGE_SIZE,
318 VM_PROT_READ|VM_PROT_WRITE, 0);
319 }
320 }
321 pmap_update(pmap_kernel());
322 initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
323
324 /*
325 * Allocate a submap for physio
326 */
327 minaddr = 0;
328 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
329 VM_PHYS_SIZE, 0, false, NULL);
330
331 banner();
332
333 /*
334 * This is actually done by initarm_common, but not all ports use it
335 * yet so do it here to catch them as well
336 */
337 struct lwp * const l = &lwp0;
338 struct pcb * const pcb = lwp_getpcb(l);
339
340 /* Zero out the PCB. */
341 memset(pcb, 0, sizeof(*pcb));
342
343 pcb->pcb_ksp = uvm_lwp_getuarea(l) + USPACE_SVC_STACK_TOP;
344 pcb->pcb_ksp -= sizeof(struct trapframe);
345
346 struct trapframe * tf = (struct trapframe *)pcb->pcb_ksp;
347
348 /* Zero out the trapframe. */
349 memset(tf, 0, sizeof(*tf));
350 lwp_settrapframe(l, tf);
351
352 tf->tf_spsr = PSR_USR32_MODE;
353 #ifdef _ARM_ARCH_BE8
354 tf->tf_spsr |= PSR_E_BIT;
355 #endif
356
357 cpu_startup_hook();
358 }
359
__weak_alias(cpu_startup_hook,cpu_startup_default)360 __weak_alias(cpu_startup_hook,cpu_startup_default)
361 void
362 cpu_startup_default(void)
363 {
364 }
365
366 /*
367 * machine dependent system variables.
368 */
369 static int
sysctl_machdep_booted_device(SYSCTLFN_ARGS)370 sysctl_machdep_booted_device(SYSCTLFN_ARGS)
371 {
372 struct sysctlnode node;
373
374 if (booted_device == NULL)
375 return EOPNOTSUPP;
376
377 node = *rnode;
378 node.sysctl_data = __UNCONST(device_xname(booted_device));
379 node.sysctl_size = strlen(device_xname(booted_device)) + 1;
380 return sysctl_lookup(SYSCTLFN_CALL(&node));
381 }
382
383 static int
sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)384 sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)
385 {
386 struct sysctlnode node;
387
388 if (booted_kernel == NULL || booted_kernel[0] == '\0')
389 return EOPNOTSUPP;
390
391 node = *rnode;
392 node.sysctl_data = booted_kernel;
393 node.sysctl_size = strlen(booted_kernel) + 1;
394 return sysctl_lookup(SYSCTLFN_CALL(&node));
395 }
396
397 static int
sysctl_machdep_cpu_arch(SYSCTLFN_ARGS)398 sysctl_machdep_cpu_arch(SYSCTLFN_ARGS)
399 {
400 struct sysctlnode node = *rnode;
401 node.sysctl_data = __UNCONST(cpu_arch);
402 node.sysctl_size = strlen(cpu_arch) + 1;
403 return sysctl_lookup(SYSCTLFN_CALL(&node));
404 }
405
406 static int
sysctl_machdep_powersave(SYSCTLFN_ARGS)407 sysctl_machdep_powersave(SYSCTLFN_ARGS)
408 {
409 struct sysctlnode node = *rnode;
410 int error, newval;
411
412 newval = cpu_do_powersave;
413 node.sysctl_data = &newval;
414 if (cpufuncs.cf_sleep == (void *) cpufunc_nullop)
415 node.sysctl_flags &= ~CTLFLAG_READWRITE;
416 error = sysctl_lookup(SYSCTLFN_CALL(&node));
417 if (error || newp == NULL || newval == cpu_do_powersave)
418 return error;
419
420 if (newval < 0 || newval > 1)
421 return EINVAL;
422 cpu_do_powersave = newval;
423
424 return 0;
425 }
426
427 SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
428 {
429
430 sysctl_createv(clog, 0, NULL, NULL,
431 CTLFLAG_PERMANENT,
432 CTLTYPE_NODE, "machdep", NULL,
433 NULL, 0, NULL, 0,
434 CTL_MACHDEP, CTL_EOL);
435
436 sysctl_createv(clog, 0, NULL, NULL,
437 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
438 CTLTYPE_INT, "debug", NULL,
439 NULL, 0, &kernel_debug, 0,
440 CTL_MACHDEP, CPU_DEBUG, CTL_EOL);
441 sysctl_createv(clog, 0, NULL, NULL,
442 CTLFLAG_PERMANENT,
443 CTLTYPE_STRING, "booted_device", NULL,
444 sysctl_machdep_booted_device, 0, NULL, 0,
445 CTL_MACHDEP, CPU_BOOTED_DEVICE, CTL_EOL);
446 sysctl_createv(clog, 0, NULL, NULL,
447 CTLFLAG_PERMANENT,
448 CTLTYPE_STRING, "booted_kernel", NULL,
449 sysctl_machdep_booted_kernel, 0, NULL, 0,
450 CTL_MACHDEP, CPU_BOOTED_KERNEL, CTL_EOL);
451 sysctl_createv(clog, 0, NULL, NULL,
452 CTLFLAG_PERMANENT,
453 CTLTYPE_STRUCT, "console_device", NULL,
454 sysctl_consdev, 0, NULL, sizeof(dev_t),
455 CTL_MACHDEP, CPU_CONSDEV, CTL_EOL);
456 sysctl_createv(clog, 0, NULL, NULL,
457 CTLFLAG_PERMANENT,
458 CTLTYPE_STRING, "cpu_arch", NULL,
459 sysctl_machdep_cpu_arch, 0, NULL, 0,
460 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
461 sysctl_createv(clog, 0, NULL, NULL,
462 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
463 CTLTYPE_INT, "powersave", NULL,
464 sysctl_machdep_powersave, 0, &cpu_do_powersave, 0,
465 CTL_MACHDEP, CPU_POWERSAVE, CTL_EOL);
466 sysctl_createv(clog, 0, NULL, NULL,
467 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
468 CTLTYPE_INT, "cpu_id", NULL,
469 NULL, curcpu()->ci_arm_cpuid, NULL, 0,
470 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
471 #ifdef FPU_VFP
472 sysctl_createv(clog, 0, NULL, NULL,
473 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
474 CTLTYPE_INT, "fpu_id", NULL,
475 NULL, 0, &cpu_info_store[0].ci_vfp_id, 0,
476 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
477 #endif
478 sysctl_createv(clog, 0, NULL, NULL,
479 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
480 CTLTYPE_INT, "fpu_present", NULL,
481 NULL, 0, &cpu_fpu_present, 0,
482 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
483 sysctl_createv(clog, 0, NULL, NULL,
484 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
485 CTLTYPE_INT, "hwdiv_present", NULL,
486 NULL, 0, &cpu_hwdiv_present, 0,
487 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
488 sysctl_createv(clog, 0, NULL, NULL,
489 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
490 CTLTYPE_INT, "neon_present", NULL,
491 NULL, 0, &cpu_neon_present, 0,
492 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
493 sysctl_createv(clog, 0, NULL, NULL,
494 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
495 CTLTYPE_STRUCT, "id_isar", NULL,
496 NULL, 0,
497 cpu_instruction_set_attributes,
498 sizeof(cpu_instruction_set_attributes),
499 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
500 sysctl_createv(clog, 0, NULL, NULL,
501 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
502 CTLTYPE_STRUCT, "id_mmfr", NULL,
503 NULL, 0,
504 cpu_memory_model_features,
505 sizeof(cpu_memory_model_features),
506 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
507 sysctl_createv(clog, 0, NULL, NULL,
508 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
509 CTLTYPE_STRUCT, "id_pfr", NULL,
510 NULL, 0,
511 cpu_processor_features,
512 sizeof(cpu_processor_features),
513 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
514 sysctl_createv(clog, 0, NULL, NULL,
515 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
516 CTLTYPE_STRUCT, "id_mvfr", NULL,
517 NULL, 0,
518 cpu_media_and_vfp_features,
519 sizeof(cpu_media_and_vfp_features),
520 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
521 sysctl_createv(clog, 0, NULL, NULL,
522 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
523 CTLTYPE_INT, "simd_present", NULL,
524 NULL, 0, &cpu_simd_present, 0,
525 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
526 sysctl_createv(clog, 0, NULL, NULL,
527 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
528 CTLTYPE_INT, "simdex_present", NULL,
529 NULL, 0, &cpu_simdex_present, 0,
530 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
531 sysctl_createv(clog, 0, NULL, NULL,
532 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
533 CTLTYPE_INT, "synchprim_present", NULL,
534 NULL, 0, &cpu_synchprim_present, 0,
535 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
536 sysctl_createv(clog, 0, NULL, NULL,
537 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
538 CTLTYPE_INT, "printfataltraps", NULL,
539 NULL, 0, &cpu_printfataltraps, 0,
540 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
541 cpu_unaligned_sigbus =
542 #if defined(__ARMEL__)
543 !CPU_IS_ARMV6_P() && !CPU_IS_ARMV7_P();
544 #elif defined(_ARM_ARCH_BE8)
545 0;
546 #else
547 1;
548 #endif
549 sysctl_createv(clog, 0, NULL, NULL,
550 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
551 CTLTYPE_INT, "unaligned_sigbus",
552 SYSCTL_DESCR("Do SIGBUS for fixed unaligned accesses"),
553 NULL, 0, &cpu_unaligned_sigbus, 0,
554 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
555 }
556
557 void
parse_mi_bootargs(char * args)558 parse_mi_bootargs(char *args)
559 {
560 int integer;
561
562 if (get_bootconf_option(args, "-1", BOOTOPT_TYPE_BOOLEAN, &integer))
563 if (integer)
564 boothowto |= RB_MD1;
565 if (get_bootconf_option(args, "single", BOOTOPT_TYPE_BOOLEAN, &integer)
566 || get_bootconf_option(args, "-s", BOOTOPT_TYPE_BOOLEAN, &integer))
567 if (integer)
568 boothowto |= RB_SINGLE;
569 if (get_bootconf_option(args, "kdb", BOOTOPT_TYPE_BOOLEAN, &integer)
570 || get_bootconf_option(args, "-k", BOOTOPT_TYPE_BOOLEAN, &integer)
571 || get_bootconf_option(args, "-d", BOOTOPT_TYPE_BOOLEAN, &integer))
572 if (integer)
573 boothowto |= RB_KDB;
574 if (get_bootconf_option(args, "ask", BOOTOPT_TYPE_BOOLEAN, &integer)
575 || get_bootconf_option(args, "-a", BOOTOPT_TYPE_BOOLEAN, &integer))
576 if (integer)
577 boothowto |= RB_ASKNAME;
578 if (get_bootconf_option(args, "userconf", BOOTOPT_TYPE_BOOLEAN, &integer)
579 || get_bootconf_option(args, "-c", BOOTOPT_TYPE_BOOLEAN, &integer))
580 if (integer)
581 boothowto |= RB_USERCONF;
582 if (get_bootconf_option(args, "halt", BOOTOPT_TYPE_BOOLEAN, &integer)
583 || get_bootconf_option(args, "-b", BOOTOPT_TYPE_BOOLEAN, &integer))
584 if (integer)
585 boothowto |= RB_HALT;
586 if (get_bootconf_option(args, "-1", BOOTOPT_TYPE_BOOLEAN, &integer))
587 if (integer)
588 boothowto |= RB_MD1;
589 if (get_bootconf_option(args, "-2", BOOTOPT_TYPE_BOOLEAN, &integer))
590 if (integer)
591 boothowto |= RB_MD2;
592 if (get_bootconf_option(args, "-3", BOOTOPT_TYPE_BOOLEAN, &integer))
593 if (integer)
594 boothowto |= RB_MD3;
595 if (get_bootconf_option(args, "-4", BOOTOPT_TYPE_BOOLEAN, &integer))
596 if (integer)
597 boothowto |= RB_MD4;
598
599 /* if (get_bootconf_option(args, "nbuf", BOOTOPT_TYPE_INT, &integer))
600 bufpages = integer;*/
601
602 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
603 if (get_bootconf_option(args, "memorydisc", BOOTOPT_TYPE_INT, &integer)
604 || get_bootconf_option(args, "memorydisk", BOOTOPT_TYPE_INT, &integer)) {
605 md_root_size = integer;
606 md_root_size *= 1024;
607 if (md_root_size < 32*1024)
608 md_root_size = 32*1024;
609 if (md_root_size > 2048*1024)
610 md_root_size = 2048*1024;
611 }
612 #endif /* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
613
614 if (get_bootconf_option(args, "quiet", BOOTOPT_TYPE_BOOLEAN, &integer)
615 || get_bootconf_option(args, "-q", BOOTOPT_TYPE_BOOLEAN, &integer))
616 if (integer)
617 boothowto |= AB_QUIET;
618 if (get_bootconf_option(args, "verbose", BOOTOPT_TYPE_BOOLEAN, &integer)
619 || get_bootconf_option(args, "-v", BOOTOPT_TYPE_BOOLEAN, &integer))
620 if (integer)
621 boothowto |= AB_VERBOSE;
622 if (get_bootconf_option(args, "debug", BOOTOPT_TYPE_BOOLEAN, &integer)
623 || get_bootconf_option(args, "-x", BOOTOPT_TYPE_BOOLEAN, &integer))
624 if (integer)
625 boothowto |= AB_DEBUG;
626 if (get_bootconf_option(args, "silent", BOOTOPT_TYPE_BOOLEAN, &integer)
627 || get_bootconf_option(args, "-z", BOOTOPT_TYPE_BOOLEAN, &integer))
628 if (integer)
629 boothowto |= AB_SILENT;
630 }
631
632 #ifdef __HAVE_FAST_SOFTINTS
633 #if IPL_SOFTSERIAL != IPL_SOFTNET + 1
634 #error IPLs are screwed up
635 #elif IPL_SOFTNET != IPL_SOFTBIO + 1
636 #error IPLs are screwed up
637 #elif IPL_SOFTBIO != IPL_SOFTCLOCK + 1
638 #error IPLs are screwed up
639 #elif !(IPL_SOFTCLOCK > IPL_NONE)
640 #error IPLs are screwed up
641 #elif (IPL_NONE != 0)
642 #error IPLs are screwed up
643 #endif
644
645 #ifndef __HAVE_PIC_FAST_SOFTINTS
646 #define SOFTINT2IPLMAP \
647 (((IPL_SOFTSERIAL - IPL_SOFTCLOCK) << (SOFTINT_SERIAL * 4)) | \
648 ((IPL_SOFTNET - IPL_SOFTCLOCK) << (SOFTINT_NET * 4)) | \
649 ((IPL_SOFTBIO - IPL_SOFTCLOCK) << (SOFTINT_BIO * 4)) | \
650 ((IPL_SOFTCLOCK - IPL_SOFTCLOCK) << (SOFTINT_CLOCK * 4)))
651 #define SOFTINT2IPL(l) ((SOFTINT2IPLMAP >> ((l) * 4)) & 0x0f)
652
653 /*
654 * This returns a mask of softint IPLs that be dispatch at <ipl>
655 * SOFTIPLMASK(IPL_NONE) = 0x0000000f
656 * SOFTIPLMASK(IPL_SOFTCLOCK) = 0x0000000e
657 * SOFTIPLMASK(IPL_SOFTBIO) = 0x0000000c
658 * SOFTIPLMASK(IPL_SOFTNET) = 0x00000008
659 * SOFTIPLMASK(IPL_SOFTSERIAL) = 0x00000000
660 */
661 #define SOFTIPLMASK(ipl) ((0x0f << (ipl)) & 0x0f)
662
663 void softint_switch(lwp_t *, int);
664
665 void
softint_trigger(uintptr_t mask)666 softint_trigger(uintptr_t mask)
667 {
668 curcpu()->ci_softints |= mask;
669 }
670
671 void
softint_init_md(lwp_t * l,u_int level,uintptr_t * machdep)672 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
673 {
674 lwp_t ** lp = &l->l_cpu->ci_softlwps[level];
675 KASSERT(*lp == NULL || *lp == l);
676 *lp = l;
677 *machdep = 1 << SOFTINT2IPL(level);
678 KASSERT(level != SOFTINT_CLOCK || *machdep == (1 << (IPL_SOFTCLOCK - IPL_SOFTCLOCK)));
679 KASSERT(level != SOFTINT_BIO || *machdep == (1 << (IPL_SOFTBIO - IPL_SOFTCLOCK)));
680 KASSERT(level != SOFTINT_NET || *machdep == (1 << (IPL_SOFTNET - IPL_SOFTCLOCK)));
681 KASSERT(level != SOFTINT_SERIAL || *machdep == (1 << (IPL_SOFTSERIAL - IPL_SOFTCLOCK)));
682 }
683
684 void
dosoftints(void)685 dosoftints(void)
686 {
687 struct cpu_info * const ci = curcpu();
688 const int opl = ci->ci_cpl;
689 const uint32_t softiplmask = SOFTIPLMASK(opl);
690 int s;
691
692 s = splhigh();
693 KASSERT(s == opl);
694 for (;;) {
695 u_int softints = ci->ci_softints & softiplmask;
696 KASSERT((softints != 0) == ((ci->ci_softints >> opl) != 0));
697 KASSERT(opl == IPL_NONE || (softints & (1 << (opl - IPL_SOFTCLOCK))) == 0);
698 if (softints == 0) {
699 break;
700 }
701 #define DOSOFTINT(n) \
702 if (ci->ci_softints & (1 << (IPL_SOFT ## n - IPL_SOFTCLOCK))) { \
703 ci->ci_softints &= \
704 ~(1 << (IPL_SOFT ## n - IPL_SOFTCLOCK)); \
705 softint_switch(ci->ci_softlwps[SOFTINT_ ## n], \
706 IPL_SOFT ## n); \
707 continue; \
708 }
709 DOSOFTINT(SERIAL);
710 DOSOFTINT(NET);
711 DOSOFTINT(BIO);
712 DOSOFTINT(CLOCK);
713 panic("dosoftints wtf (softints=%u?, ipl=%d)", softints, opl);
714 }
715 splx(s);
716 }
717 #endif /* !__HAVE_PIC_FAST_SOFTINTS */
718 #endif /* __HAVE_FAST_SOFTINTS */
719
720 #ifdef MODULAR
721 /*
722 * Push any modules loaded by the boot loader.
723 */
724 void
module_init_md(void)725 module_init_md(void)
726 {
727 #ifdef FDT
728 arm_fdt_module_init();
729 #endif
730 }
731 #endif /* MODULAR */
732
733 int
mm_md_physacc(paddr_t pa,vm_prot_t prot)734 mm_md_physacc(paddr_t pa, vm_prot_t prot)
735 {
736 if (pa >= physical_start && pa < physical_end)
737 return 0;
738
739 return kauth_authorize_machdep(kauth_cred_get(),
740 KAUTH_MACHDEP_UNMANAGEDMEM, NULL, NULL, NULL, NULL);
741 }
742
743 #ifdef __HAVE_CPU_UAREA_ALLOC_IDLELWP
744 vaddr_t
cpu_uarea_alloc_idlelwp(struct cpu_info * ci)745 cpu_uarea_alloc_idlelwp(struct cpu_info *ci)
746 {
747 const vaddr_t va = idlestack.pv_va + cpu_index(ci) * USPACE;
748 // printf("%s: %s: va=%lx\n", __func__, ci->ci_data.cpu_name, va);
749 return va;
750 }
751 #endif
752
753 #ifdef MULTIPROCESSOR
754 /*
755 * Initialise a secondary processor.
756 *
757 * printf isn't available to us for a number of reasons.
758 *
759 * - kprint_init has been called and printf will try to take locks which we
760 * can't do just yet because bootstrap translation tables do not allowing
761 * caching.
762 *
763 * - kmutex(9) relies on curcpu which isn't setup yet.
764 *
765 */
766 void __noasan
cpu_init_secondary_processor(int cpuindex)767 cpu_init_secondary_processor(int cpuindex)
768 {
769 // pmap_kernel has been successfully built and we can switch to it
770 cpu_domains(DOMAIN_DEFAULT);
771 cpu_idcache_wbinv_all();
772
773 VPRINTS("index: ");
774 VPRINTX(cpuindex);
775 VPRINTS(" ttb");
776
777 cpu_setup(boot_args);
778
779 #ifdef ARM_MMU_EXTENDED
780 /*
781 * TTBCR should have been initialized by the MD start code.
782 */
783 KASSERT((armreg_contextidr_read() & 0xff) == 0);
784 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N));
785 /*
786 * Disable lookups via TTBR0 until there is an activated pmap.
787 */
788
789 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0);
790 cpu_setttb(pmap_kernel()->pm_l1_pa , KERNEL_PID);
791 isb();
792 #else
793 cpu_setttb(pmap_kernel()->pm_l1->l1_physaddr, true);
794 #endif
795
796 cpu_tlb_flushID();
797
798 VPRINTS(" (TTBR0=");
799 VPRINTX(armreg_ttbr_read());
800 VPRINTS(")");
801
802 #ifdef ARM_MMU_EXTENDED
803 VPRINTS(" (TTBR1=");
804 VPRINTX(armreg_ttbr1_read());
805 VPRINTS(")");
806 VPRINTS(" (TTBCR=");
807 VPRINTX(armreg_ttbcr_read());
808 VPRINTS(")");
809 #endif
810
811 struct cpu_info * ci = &cpu_info_store[cpuindex];
812
813 VPRINTS(" ci = ");
814 VPRINTX((int)ci);
815
816 ci->ci_ctrl = armreg_sctlr_read();
817 ci->ci_arm_cpuid = cpu_idnum();
818 ci->ci_arm_cputype = ci->ci_arm_cpuid & CPU_ID_CPU_MASK;
819 ci->ci_arm_cpurev = ci->ci_arm_cpuid & CPU_ID_REVISION_MASK;
820
821 ci->ci_midr = armreg_midr_read();
822 ci->ci_actlr = armreg_auxctl_read();
823 ci->ci_revidr = armreg_revidr_read();
824 ci->ci_mpidr = armreg_mpidr_read();
825
826 arm_cpu_topology_set(ci, ci->ci_mpidr);
827
828 VPRINTS(" vfp");
829 vfp_detect(ci);
830
831 VPRINTS(" hatched |=");
832 VPRINTX(__BIT(cpuindex));
833 VPRINTS("\n\r");
834
835 cpu_set_hatched(cpuindex);
836
837 /*
838 * return to assembly to wait for cpu_boot_secondary_processors
839 */
840 }
841
842 void
xc_send_ipi(struct cpu_info * ci)843 xc_send_ipi(struct cpu_info *ci)
844 {
845 KASSERT(kpreempt_disabled());
846 KASSERT(curcpu() != ci);
847
848 intr_ipi_send(ci != NULL ? ci->ci_kcpuset : NULL, IPI_XCALL);
849 }
850
851 void
cpu_ipi(struct cpu_info * ci)852 cpu_ipi(struct cpu_info *ci)
853 {
854 KASSERT(kpreempt_disabled());
855 KASSERT(curcpu() != ci);
856
857 intr_ipi_send(ci != NULL ? ci->ci_kcpuset : NULL, IPI_GENERIC);
858 }
859
860 #endif /* MULTIPROCESSOR */
861
862 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
863 bool
mm_md_direct_mapped_phys(paddr_t pa,vaddr_t * vap)864 mm_md_direct_mapped_phys(paddr_t pa, vaddr_t *vap)
865 {
866 bool rv;
867 vaddr_t va = pmap_direct_mapped_phys(pa, &rv, 0);
868 if (rv) {
869 *vap = va;
870 }
871 return rv;
872 }
873 #endif
874
875 bool
mm_md_page_color(paddr_t pa,int * colorp)876 mm_md_page_color(paddr_t pa, int *colorp)
877 {
878 #if (ARM_MMU_V6 + ARM_MMU_V7) != 0
879 *colorp = atop(pa & arm_cache_prefer_mask);
880
881 return arm_cache_prefer_mask ? false : true;
882 #else
883 *colorp = 0;
884
885 return true;
886 #endif
887 }
888
889 #if defined(FDT)
890 extern char KERNEL_BASE_phys[];
891 #define KERNEL_BASE_PHYS ((paddr_t)KERNEL_BASE_phys)
892
893 void
cpu_kernel_vm_init(paddr_t memory_start,psize_t memory_size)894 cpu_kernel_vm_init(paddr_t memory_start, psize_t memory_size)
895 {
896 const struct fdt_platform *plat = fdt_platform_find();
897
898 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
899 const bool mapallmem_p = true;
900 #ifndef PMAP_NEED_ALLOC_POOLPAGE
901 if (memory_size > KERNEL_VM_BASE - KERNEL_BASE) {
902 VPRINTF("%s: dropping RAM size from %luMB to %uMB\n",
903 __func__, (unsigned long) (memory_size >> 20),
904 (KERNEL_VM_BASE - KERNEL_BASE) >> 20);
905 memory_size = KERNEL_VM_BASE - KERNEL_BASE;
906 }
907 #endif
908 #else
909 const bool mapallmem_p = false;
910 #endif
911
912 VPRINTF("%s: kernel phys start %" PRIxPADDR " end %" PRIxPADDR "\n",
913 __func__, memory_start, memory_start + memory_size);
914
915 arm32_bootmem_init(memory_start, memory_size, KERNEL_BASE_PHYS);
916 arm32_kernel_vm_init(KERNEL_VM_BASE, ARM_VECTORS_HIGH, 0,
917 plat->fp_devmap(), mapallmem_p);
918 }
919 #endif
920
921