1 /* $OpenBSD: machdep.c,v 1.95 2024/11/18 05:32:39 jsg Exp $ */
2 /*
3 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se>
4 * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/param.h>
20 #include <sys/systm.h>
21 #include <sys/sched.h>
22 #include <sys/proc.h>
23 #include <sys/sysctl.h>
24 #include <sys/reboot.h>
25 #include <sys/mount.h>
26 #include <sys/exec.h>
27 #include <sys/user.h>
28 #include <sys/conf.h>
29 #include <sys/kcore.h>
30 #include <sys/core.h>
31 #include <sys/msgbuf.h>
32 #include <sys/buf.h>
33 #include <sys/termios.h>
34 #include <sys/sensors.h>
35 #include <sys/malloc.h>
36
37 #include <net/if.h>
38 #include <uvm/uvm_extern.h>
39 #include <dev/cons.h>
40 #include <dev/ofw/fdt.h>
41 #include <dev/ofw/openfirm.h>
42 #include <machine/param.h>
43 #include <machine/kcore.h>
44 #include <machine/bootconfig.h>
45 #include <machine/bus.h>
46 #include <machine/fpu.h>
47
48 #include <machine/db_machdep.h>
49 #include <ddb/db_extern.h>
50
51 #include <dev/efi/efi.h>
52
53 #include "softraid.h"
54 #if NSOFTRAID > 0
55 #include <dev/softraidvar.h>
56 #endif
57
58 extern vaddr_t virtual_avail;
59 extern uint64_t esym;
60
61 extern char _start[];
62
63 char *boot_args = NULL;
64 uint8_t *bootmac = NULL;
65
66 int stdout_node;
67 int stdout_speed;
68
69 void (*cpuresetfn)(void);
70 void (*powerdownfn)(void);
71
72 int cold = 1;
73 int lid_action = 1;
74
75 struct vm_map *exec_map = NULL;
76 struct vm_map *phys_map = NULL;
77
78 int physmem;
79
80 struct consdev *cn_tab;
81
82 caddr_t msgbufaddr;
83 paddr_t msgbufphys;
84
85 struct user *proc0paddr;
86
87 struct uvm_constraint_range dma_constraint = { 0x0, (paddr_t)-1 };
88 struct uvm_constraint_range *uvm_md_constraints[] = {
89 &dma_constraint,
90 NULL,
91 };
92
93 /* the following is used externally (sysctl_hw) */
94 char machine[] = MACHINE; /* from <machine/param.h> */
95
96 int safepri = 0;
97
98 struct cpu_info cpu_info_primary;
99 struct cpu_info *cpu_info[MAXCPUS] = { &cpu_info_primary };
100
101 struct fdt_reg memreg[VM_PHYSSEG_MAX];
102 int nmemreg;
103
104 void memreg_add(const struct fdt_reg *);
105 void memreg_remove(const struct fdt_reg *);
106
107 static int
atoi(const char * s)108 atoi(const char *s)
109 {
110 int n, neg;
111
112 n = 0;
113 neg = 0;
114
115 while (*s == '-') {
116 s++;
117 neg = !neg;
118 }
119
120 while (*s != '\0') {
121 if (*s < '0' || *s > '9')
122 break;
123
124 n = (10 * n) + (*s - '0');
125 s++;
126 }
127
128 return (neg ? -n : n);
129 }
130
131 void *
fdt_find_cons(const char * name)132 fdt_find_cons(const char *name)
133 {
134 char *alias = "serial0";
135 char buf[128];
136 char *stdout = NULL;
137 char *p;
138 void *node;
139
140 /* First check if "stdout-path" is set. */
141 node = fdt_find_node("/chosen");
142 if (node) {
143 if (fdt_node_property(node, "stdout-path", &stdout) > 0) {
144 if (strchr(stdout, ':') != NULL) {
145 strlcpy(buf, stdout, sizeof(buf));
146 if ((p = strchr(buf, ':')) != NULL) {
147 *p++ = '\0';
148 stdout_speed = atoi(p);
149 }
150 stdout = buf;
151 }
152 if (stdout[0] != '/') {
153 /* It's an alias. */
154 alias = stdout;
155 stdout = NULL;
156 }
157 }
158 }
159
160 /* Perform alias lookup if necessary. */
161 if (stdout == NULL) {
162 node = fdt_find_node("/aliases");
163 if (node)
164 fdt_node_property(node, alias, &stdout);
165 }
166
167 /* Lookup the physical address of the interface. */
168 if (stdout) {
169 node = fdt_find_node(stdout);
170 if (node && fdt_is_compatible(node, name)) {
171 stdout_node = OF_finddevice(stdout);
172 return (node);
173 }
174 }
175
176 return (NULL);
177 }
178
179 void amluart_init_cons(void);
180 void cduart_init_cons(void);
181 void com_fdt_init_cons(void);
182 void exuart_init_cons(void);
183 void imxuart_init_cons(void);
184 void mvuart_init_cons(void);
185 void pluart_init_cons(void);
186 void simplefb_init_cons(bus_space_tag_t);
187
188 void
consinit(void)189 consinit(void)
190 {
191 static int consinit_called = 0;
192
193 if (consinit_called != 0)
194 return;
195
196 consinit_called = 1;
197
198 amluart_init_cons();
199 cduart_init_cons();
200 com_fdt_init_cons();
201 exuart_init_cons();
202 imxuart_init_cons();
203 mvuart_init_cons();
204 pluart_init_cons();
205 simplefb_init_cons(&arm64_bs_tag);
206 }
207
208 void
cpu_idle_enter(void)209 cpu_idle_enter(void)
210 {
211 disable_irq_daif();
212 }
213
214 void (*cpu_idle_cycle_fcn)(void) = cpu_wfi;
215
216 void
cpu_idle_cycle(void)217 cpu_idle_cycle(void)
218 {
219 cpu_idle_cycle_fcn();
220 enable_irq_daif();
221 disable_irq_daif();
222 }
223
224 void
cpu_idle_leave(void)225 cpu_idle_leave(void)
226 {
227 enable_irq_daif();
228 }
229
230 /* Dummy trapframe for proc0. */
231 struct trapframe proc0tf;
232
233 void
cpu_startup(void)234 cpu_startup(void)
235 {
236 u_int loop;
237 paddr_t minaddr;
238 paddr_t maxaddr;
239
240 proc0.p_addr = proc0paddr;
241
242 /*
243 * Give pmap a chance to set up a few more things now the vm
244 * is initialised
245 */
246 pmap_postinit();
247
248 /*
249 * Initialize error message buffer (at end of core).
250 */
251
252 /* msgbufphys was setup during the secondary boot strap */
253 for (loop = 0; loop < atop(MSGBUFSIZE); ++loop)
254 pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
255 msgbufphys + loop * PAGE_SIZE, PROT_READ | PROT_WRITE);
256 pmap_update(pmap_kernel());
257 initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
258
259 /*
260 * Identify ourselves for the msgbuf (everything printed earlier will
261 * not be buffered).
262 */
263 printf("%s", version);
264
265 printf("real mem = %lu (%luMB)\n", ptoa(physmem),
266 ptoa(physmem) / 1024 / 1024);
267
268 /*
269 * Allocate a submap for exec arguments. This map effectively
270 * limits the number of processes exec'ing at any time.
271 */
272 minaddr = vm_map_min(kernel_map);
273 exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
274 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
275
276 /*
277 * Allocate a submap for physio
278 */
279 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
280 VM_PHYS_SIZE, 0, FALSE, NULL);
281
282 /*
283 * Set up buffers, so they can be used to read disk labels.
284 */
285 bufinit();
286
287 printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free),
288 ptoa(uvmexp.free) / 1024 / 1024);
289
290 curpcb = &proc0.p_addr->u_pcb;
291 curpcb->pcb_flags = 0;
292 curpcb->pcb_tf = &proc0tf;
293
294 if (boothowto & RB_CONFIG) {
295 #ifdef BOOT_CONFIG
296 user_config();
297 #else
298 printf("kernel does not support -c; continuing..\n");
299 #endif
300 }
301 }
302
303 void cpu_switchto_asm(struct proc *, struct proc *);
304
305 void
cpu_switchto(struct proc * old,struct proc * new)306 cpu_switchto(struct proc *old, struct proc *new)
307 {
308 if (old) {
309 struct pcb *pcb = &old->p_addr->u_pcb;
310
311 if (pcb->pcb_flags & PCB_FPU)
312 fpu_save(old);
313
314 fpu_drop();
315 }
316
317 cpu_switchto_asm(old, new);
318 }
319
320 /*
321 * machine dependent system variables.
322 */
323
324 const struct sysctl_bounded_args cpuctl_vars[] = {
325 { CPU_LIDACTION, &lid_action, 0, 2 },
326 };
327
328 int
cpu_sysctl(int * name,u_int namelen,void * oldp,size_t * oldlenp,void * newp,size_t newlen,struct proc * p)329 cpu_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
330 size_t newlen, struct proc *p)
331 {
332 char *compatible;
333 int node, len, error;
334
335 /* all sysctl names at this level are terminal */
336 if (namelen != 1)
337 return (ENOTDIR); /* overloaded */
338
339 switch (name[0]) {
340 case CPU_COMPATIBLE:
341 node = OF_finddevice("/");
342 len = OF_getproplen(node, "compatible");
343 if (len <= 0)
344 return (EOPNOTSUPP);
345 compatible = malloc(len, M_TEMP, M_WAITOK | M_ZERO);
346 OF_getprop(node, "compatible", compatible, len);
347 compatible[len - 1] = 0;
348 error = sysctl_rdstring(oldp, oldlenp, newp, compatible);
349 free(compatible, M_TEMP, len);
350 return error;
351 case CPU_ID_AA64ISAR0:
352 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64isar0);
353 case CPU_ID_AA64ISAR1:
354 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64isar1);
355 case CPU_ID_AA64ISAR2:
356 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64isar2);
357 case CPU_ID_AA64PFR0:
358 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64pfr0);
359 case CPU_ID_AA64PFR1:
360 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64pfr1);
361 case CPU_ID_AA64MMFR0:
362 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64mmfr0);
363 case CPU_ID_AA64MMFR1:
364 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64mmfr1);
365 case CPU_ID_AA64MMFR2:
366 return sysctl_rdquad(oldp, oldlenp, newp, cpu_id_aa64mmfr2);
367 case CPU_ID_AA64SMFR0:
368 case CPU_ID_AA64ZFR0:
369 return sysctl_rdquad(oldp, oldlenp, newp, 0);
370 default:
371 return (sysctl_bounded_arr(cpuctl_vars, nitems(cpuctl_vars),
372 name, namelen, oldp, oldlenp, newp, newlen));
373 }
374 /* NOTREACHED */
375 }
376
377 void dumpsys(void);
378
379 int waittime = -1;
380
381 __dead void
boot(int howto)382 boot(int howto)
383 {
384 if ((howto & RB_RESET) != 0)
385 goto doreset;
386
387 if (cold) {
388 if ((howto & RB_USERREQ) == 0)
389 howto |= RB_HALT;
390 goto haltsys;
391 }
392
393 boothowto = howto;
394 if ((howto & RB_NOSYNC) == 0 && waittime < 0) {
395 waittime = 0;
396 vfs_shutdown(curproc);
397
398 if ((howto & RB_TIMEBAD) == 0) {
399 resettodr();
400 } else {
401 printf("WARNING: not updating battery clock\n");
402 }
403 }
404 if_downall();
405
406 uvm_shutdown();
407 splhigh();
408 cold = 1;
409
410 if ((howto & RB_DUMP) != 0)
411 dumpsys();
412
413 haltsys:
414 config_suspend_all(DVACT_POWERDOWN);
415
416 if ((howto & RB_HALT) != 0) {
417 if ((howto & RB_POWERDOWN) != 0) {
418 printf("\nAttempting to power down...\n");
419 delay(500000);
420 if (powerdownfn)
421 (*powerdownfn)();
422 }
423
424 printf("\n");
425 printf("The operating system has halted.\n");
426 printf("Please press any key to reboot.\n\n");
427 cngetc();
428 }
429
430 doreset:
431 printf("rebooting...\n");
432 delay(500000);
433 if (cpuresetfn)
434 (*cpuresetfn)();
435 printf("reboot failed; spinning\n");
436 for (;;)
437 continue;
438 /* NOTREACHED */
439 }
440
441 void
setregs(struct proc * p,struct exec_package * pack,u_long stack,struct ps_strings * arginfo)442 setregs(struct proc *p, struct exec_package *pack, u_long stack,
443 struct ps_strings *arginfo)
444 {
445 struct pmap *pm = p->p_vmspace->vm_map.pmap;
446 struct pcb *pcb = &p->p_addr->u_pcb;
447 struct trapframe *tf = pcb->pcb_tf;
448
449 if (pack->ep_flags & EXEC_NOBTCFI)
450 pm->pm_guarded = 0;
451 else
452 pm->pm_guarded = ATTR_GP;
453
454 arc4random_buf(&pm->pm_apiakey, sizeof(pm->pm_apiakey));
455 arc4random_buf(&pm->pm_apdakey, sizeof(pm->pm_apdakey));
456 arc4random_buf(&pm->pm_apibkey, sizeof(pm->pm_apibkey));
457 arc4random_buf(&pm->pm_apdbkey, sizeof(pm->pm_apdbkey));
458 arc4random_buf(&pm->pm_apgakey, sizeof(pm->pm_apgakey));
459 pmap_setpauthkeys(pm);
460
461 /* If we were using the FPU, forget about it. */
462 memset(&pcb->pcb_fpstate, 0, sizeof(pcb->pcb_fpstate));
463 pcb->pcb_flags &= ~PCB_FPU;
464 fpu_drop();
465
466 memset(tf, 0, sizeof *tf);
467 tf->tf_sp = stack;
468 tf->tf_lr = pack->ep_entry;
469 tf->tf_elr = pack->ep_entry; /* ??? */
470 tf->tf_spsr = PSR_M_EL0t | PSR_DIT;
471 }
472
473 void
need_resched(struct cpu_info * ci)474 need_resched(struct cpu_info *ci)
475 {
476 ci->ci_want_resched = 1;
477
478 /* There's a risk we'll be called before the idle threads start */
479 if (ci->ci_curproc) {
480 aston(ci->ci_curproc);
481 cpu_kick(ci);
482 }
483 }
484
485 int cpu_dumpsize(void);
486 u_long cpu_dump_mempagecnt(void);
487
488 paddr_t dumpmem_paddr;
489 vaddr_t dumpmem_vaddr;
490 psize_t dumpmem_sz;
491
492 /*
493 * These variables are needed by /sbin/savecore
494 */
495 u_long dumpmag = 0x8fca0101; /* magic number */
496 int dumpsize = 0; /* pages */
497 long dumplo = 0; /* blocks */
498
499 /*
500 * cpu_dump: dump the machine-dependent kernel core dump headers.
501 */
502 int
cpu_dump(void)503 cpu_dump(void)
504 {
505 int (*dump)(dev_t, daddr_t, caddr_t, size_t);
506 char buf[dbtob(1)];
507 kcore_seg_t *segp;
508 cpu_kcore_hdr_t *cpuhdrp;
509 phys_ram_seg_t *memsegp;
510 #if 0
511 caddr_t va;
512 int i;
513 #endif
514
515 dump = bdevsw[major(dumpdev)].d_dump;
516
517 memset(buf, 0, sizeof buf);
518 segp = (kcore_seg_t *)buf;
519 cpuhdrp = (cpu_kcore_hdr_t *)&buf[ALIGN(sizeof(*segp))];
520 memsegp = (phys_ram_seg_t *)&buf[ALIGN(sizeof(*segp)) +
521 ALIGN(sizeof(*cpuhdrp))];
522
523 /*
524 * Generate a segment header.
525 */
526 CORE_SETMAGIC(*segp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
527 segp->c_size = dbtob(1) - ALIGN(sizeof(*segp));
528
529 /*
530 * Add the machine-dependent header info.
531 */
532 cpuhdrp->kernelbase = KERNEL_BASE;
533 cpuhdrp->kerneloffs = 0;
534 cpuhdrp->staticsize = 0;
535 cpuhdrp->pmap_kernel_l1 = 0;
536 cpuhdrp->pmap_kernel_l2 = 0;
537
538 #if 0
539 /*
540 * Fill in the memory segment descriptors.
541 */
542 for (i = 0; i < mem_cluster_cnt; i++) {
543 memsegp[i].start = mem_clusters[i].start;
544 memsegp[i].size = mem_clusters[i].size & PMAP_PA_MASK;
545 }
546
547 /*
548 * If we have dump memory then assume the kernel stack is in high
549 * memory and bounce
550 */
551 if (dumpmem_vaddr != 0) {
552 memcpy((char *)dumpmem_vaddr, buf, sizeof(buf));
553 va = (caddr_t)dumpmem_vaddr;
554 } else {
555 va = (caddr_t)buf;
556 }
557 return (dump(dumpdev, dumplo, va, dbtob(1)));
558 #else
559 return ENOSYS;
560 #endif
561 }
562
563 /*
564 * This is called by main to set dumplo and dumpsize.
565 * Dumps always skip the first PAGE_SIZE of disk space
566 * in case there might be a disk label stored there.
567 * If there is extra space, put dump at the end to
568 * reduce the chance that swapping trashes it.
569 */
570 void
dumpconf(void)571 dumpconf(void)
572 {
573 int nblks, dumpblks; /* size of dump area */
574
575 if (dumpdev == NODEV ||
576 (nblks = (bdevsw[major(dumpdev)].d_psize)(dumpdev)) == 0)
577 return;
578 if (nblks <= ctod(1))
579 return;
580
581 dumpblks = cpu_dumpsize();
582 if (dumpblks < 0)
583 return;
584 dumpblks += ctod(cpu_dump_mempagecnt());
585
586 /* If dump won't fit (incl. room for possible label), punt. */
587 if (dumpblks > (nblks - ctod(1)))
588 return;
589
590 /* Put dump at end of partition */
591 dumplo = nblks - dumpblks;
592
593 /* dumpsize is in page units, and doesn't include headers. */
594 dumpsize = cpu_dump_mempagecnt();
595 }
596
597 /*
598 * Doadump comes here after turning off memory management and
599 * getting on the dump stack, either when called above, or by
600 * the auto-restart code.
601 */
602 #define BYTES_PER_DUMP MAXPHYS /* must be a multiple of pagesize */
603
604 void
dumpsys(void)605 dumpsys(void)
606 {
607 u_long totalbytesleft, bytes, i, n, memseg;
608 u_long maddr;
609 daddr_t blkno;
610 void *va;
611 int (*dump)(dev_t, daddr_t, caddr_t, size_t);
612 int error;
613
614 if (dumpdev == NODEV)
615 return;
616
617 /*
618 * For dumps during autoconfiguration,
619 * if dump device has already configured...
620 */
621 if (dumpsize == 0)
622 dumpconf();
623 if (dumplo <= 0 || dumpsize == 0) {
624 printf("\ndump to dev %u,%u not possible\n", major(dumpdev),
625 minor(dumpdev));
626 return;
627 }
628 printf("\ndumping to dev %u,%u offset %ld\n", major(dumpdev),
629 minor(dumpdev), dumplo);
630
631 error = (*bdevsw[major(dumpdev)].d_psize)(dumpdev);
632 printf("dump ");
633 if (error == -1) {
634 printf("area unavailable\n");
635 return;
636 }
637
638 if ((error = cpu_dump()) != 0)
639 goto err;
640
641 totalbytesleft = ptoa(cpu_dump_mempagecnt());
642 blkno = dumplo + cpu_dumpsize();
643 dump = bdevsw[major(dumpdev)].d_dump;
644 error = 0;
645
646 bytes = n = i = memseg = 0;
647 maddr = 0;
648 va = 0;
649 #if 0
650 for (memseg = 0; memseg < mem_cluster_cnt; memseg++) {
651 maddr = mem_clusters[memseg].start;
652 bytes = mem_clusters[memseg].size;
653
654 for (i = 0; i < bytes; i += n, totalbytesleft -= n) {
655 /* Print out how many MBs we have left to go. */
656 if ((totalbytesleft % (1024*1024)) < BYTES_PER_DUMP)
657 printf("%ld ", totalbytesleft / (1024 * 1024));
658
659 /* Limit size for next transfer. */
660 n = bytes - i;
661 if (n > BYTES_PER_DUMP)
662 n = BYTES_PER_DUMP;
663 if (maddr > 0xffffffff) {
664 va = (void *)dumpmem_vaddr;
665 if (n > dumpmem_sz)
666 n = dumpmem_sz;
667 memcpy(va, (void *)PMAP_DIRECT_MAP(maddr), n);
668 } else {
669 va = (void *)PMAP_DIRECT_MAP(maddr);
670 }
671
672 error = (*dump)(dumpdev, blkno, va, n);
673 if (error)
674 goto err;
675 maddr += n;
676 blkno += btodb(n); /* XXX? */
677
678 #if 0 /* XXX this doesn't work. grr. */
679 /* operator aborting dump? */
680 if (sget() != NULL) {
681 error = EINTR;
682 break;
683 }
684 #endif
685 }
686 }
687 #endif
688
689 err:
690 switch (error) {
691
692 case ENXIO:
693 printf("device bad\n");
694 break;
695
696 case EFAULT:
697 printf("device not ready\n");
698 break;
699
700 case EINVAL:
701 printf("area improper\n");
702 break;
703
704 case EIO:
705 printf("i/o error\n");
706 break;
707
708 case EINTR:
709 printf("aborted from console\n");
710 break;
711
712 case 0:
713 printf("succeeded\n");
714 break;
715
716 default:
717 printf("error %d\n", error);
718 break;
719 }
720 printf("\n\n");
721 delay(5000000); /* 5 seconds */
722 }
723
724
725 /*
726 * Size of memory segments, before any memory is stolen.
727 */
728 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX];
729 int mem_cluster_cnt;
730
731 /*
732 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers.
733 */
734 int
cpu_dumpsize(void)735 cpu_dumpsize(void)
736 {
737 int size;
738
739 size = ALIGN(sizeof(kcore_seg_t)) +
740 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t));
741 if (roundup(size, dbtob(1)) != dbtob(1))
742 return (-1);
743
744 return (1);
745 }
746
747 u_long
cpu_dump_mempagecnt(void)748 cpu_dump_mempagecnt(void)
749 {
750 return 0;
751 }
752
753 int64_t dcache_line_size; /* The minimum D cache line size */
754 int64_t icache_line_size; /* The minimum I cache line size */
755 int64_t idcache_line_size; /* The minimum cache line size */
756 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */
757
758 void
cache_setup(void)759 cache_setup(void)
760 {
761 int dcache_line_shift, icache_line_shift, dczva_line_shift;
762 uint32_t ctr_el0;
763 uint32_t dczid_el0;
764
765 ctr_el0 = READ_SPECIALREG(ctr_el0);
766
767 /* Read the log2 words in each D cache line */
768 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
769 /* Get the D cache line size */
770 dcache_line_size = sizeof(int) << dcache_line_shift;
771
772 /* And the same for the I cache */
773 icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
774 icache_line_size = sizeof(int) << icache_line_shift;
775
776 idcache_line_size = MIN(dcache_line_size, icache_line_size);
777
778 dczid_el0 = READ_SPECIALREG(dczid_el0);
779
780 /* Check if dc zva is not prohibited */
781 if (dczid_el0 & DCZID_DZP)
782 dczva_line_size = 0;
783 else {
784 /* Same as with above calculations */
785 dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
786 dczva_line_size = sizeof(int) << dczva_line_shift;
787 }
788 }
789
790 uint64_t mmap_start;
791 uint32_t mmap_size;
792 uint32_t mmap_desc_size;
793 uint32_t mmap_desc_ver;
794
795 EFI_MEMORY_DESCRIPTOR *mmap;
796
797 void collect_kernel_args(const char *);
798 void process_kernel_args(void);
799
800 int pmap_bootstrap_bs_map(bus_space_tag_t, bus_addr_t,
801 bus_size_t, int, bus_space_handle_t *);
802
803 void
initarm(struct arm64_bootparams * abp)804 initarm(struct arm64_bootparams *abp)
805 {
806 long kernbase = (long)_start & ~PAGE_MASK;
807 long kvo = abp->kern_delta;
808 paddr_t memstart, memend;
809 paddr_t startpa, endpa, pa;
810 vaddr_t vstart, va;
811 struct fdt_head *fh;
812 void *config = abp->arg2;
813 void *fdt = NULL;
814 struct fdt_reg reg;
815 void *node;
816 EFI_PHYSICAL_ADDRESS system_table = 0;
817 int (*map_func_save)(bus_space_tag_t, bus_addr_t, bus_size_t, int,
818 bus_space_handle_t *);
819 int i;
820
821 /*
822 * Set the per-CPU pointer with a backup in tpidr_el1 to be
823 * loaded when entering the kernel from userland.
824 */
825 __asm volatile("mov x18, %0\n"
826 "msr tpidr_el1, %0" :: "r"(&cpu_info_primary));
827
828 cache_setup();
829
830 /* The bootloader has loaded us into a 64MB block. */
831 memstart = KERNBASE + kvo;
832 memend = memstart + 64 * 1024 * 1024;
833
834 /* Bootstrap enough of pmap to enter the kernel proper. */
835 vstart = pmap_bootstrap(kvo, abp->kern_l1pt,
836 kernbase, esym, memstart, memend);
837
838 /* Map the FDT header to determine its size. */
839 va = vstart;
840 startpa = trunc_page((paddr_t)config);
841 endpa = round_page((paddr_t)config + sizeof(struct fdt_head));
842 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
843 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
844 fh = (void *)(vstart + ((paddr_t)config - startpa));
845 if (betoh32(fh->fh_magic) != FDT_MAGIC || betoh32(fh->fh_size) == 0)
846 panic("%s: no FDT", __func__);
847
848 /* Map the remainder of the FDT. */
849 endpa = round_page((paddr_t)config + betoh32(fh->fh_size));
850 for (; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
851 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
852 config = (void *)(vstart + ((paddr_t)config - startpa));
853 vstart = va;
854
855 if (!fdt_init(config))
856 panic("%s: corrupt FDT", __func__);
857
858 node = fdt_find_node("/chosen");
859 if (node != NULL) {
860 char *prop;
861 int len;
862 static uint8_t lladdr[6];
863
864 len = fdt_node_property(node, "bootargs", &prop);
865 if (len > 0)
866 collect_kernel_args(prop);
867
868 len = fdt_node_property(node, "openbsd,boothowto", &prop);
869 if (len == sizeof(boothowto))
870 boothowto = bemtoh32((uint32_t *)prop);
871
872 len = fdt_node_property(node, "openbsd,bootduid", &prop);
873 if (len == sizeof(bootduid))
874 memcpy(bootduid, prop, sizeof(bootduid));
875
876 len = fdt_node_property(node, "openbsd,bootmac", &prop);
877 if (len == sizeof(lladdr)) {
878 memcpy(lladdr, prop, sizeof(lladdr));
879 bootmac = lladdr;
880 }
881
882 len = fdt_node_property(node, "openbsd,sr-bootuuid", &prop);
883 #if NSOFTRAID > 0
884 if (len == sizeof(sr_bootuuid))
885 memcpy(&sr_bootuuid, prop, sizeof(sr_bootuuid));
886 #endif
887 if (len > 0)
888 explicit_bzero(prop, len);
889
890 len = fdt_node_property(node, "openbsd,sr-bootkey", &prop);
891 #if NSOFTRAID > 0
892 if (len == sizeof(sr_bootkey))
893 memcpy(&sr_bootkey, prop, sizeof(sr_bootkey));
894 #endif
895 if (len > 0)
896 explicit_bzero(prop, len);
897
898 len = fdt_node_property(node, "openbsd,uefi-mmap-start", &prop);
899 if (len == sizeof(mmap_start))
900 mmap_start = bemtoh64((uint64_t *)prop);
901 len = fdt_node_property(node, "openbsd,uefi-mmap-size", &prop);
902 if (len == sizeof(mmap_size))
903 mmap_size = bemtoh32((uint32_t *)prop);
904 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-size", &prop);
905 if (len == sizeof(mmap_desc_size))
906 mmap_desc_size = bemtoh32((uint32_t *)prop);
907 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-ver", &prop);
908 if (len == sizeof(mmap_desc_ver))
909 mmap_desc_ver = bemtoh32((uint32_t *)prop);
910
911 len = fdt_node_property(node, "openbsd,uefi-system-table", &prop);
912 if (len == sizeof(system_table))
913 system_table = bemtoh64((uint64_t *)prop);
914
915 len = fdt_node_property(node, "openbsd,dma-constraint", &prop);
916 if (len == sizeof(dma_constraint)) {
917 dma_constraint.ucr_low = bemtoh64((uint64_t *)prop);
918 dma_constraint.ucr_high = bemtoh64((uint64_t *)prop + 1);
919 }
920 }
921
922 process_kernel_args();
923
924 proc0paddr = (struct user *)abp->kern_stack;
925
926 msgbufaddr = (caddr_t)vstart;
927 msgbufphys = pmap_steal_avail(round_page(MSGBUFSIZE), PAGE_SIZE, NULL);
928 vstart += round_page(MSGBUFSIZE);
929
930 zero_page = vstart;
931 vstart += MAXCPUS * PAGE_SIZE;
932 copy_src_page = vstart;
933 vstart += MAXCPUS * PAGE_SIZE;
934 copy_dst_page = vstart;
935 vstart += MAXCPUS * PAGE_SIZE;
936
937 /* Relocate the FDT to safe memory. */
938 if (fdt_get_size(config) != 0) {
939 uint32_t csize, size = round_page(fdt_get_size(config));
940 paddr_t pa;
941 vaddr_t va;
942
943 pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
944 memcpy((void *)pa, config, size); /* copy to physical */
945 for (va = vstart, csize = size; csize > 0;
946 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
947 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
948
949 fdt = (void *)vstart;
950 vstart += size;
951 }
952
953 /* Relocate the EFI memory map too. */
954 if (mmap_start != 0) {
955 uint32_t csize, size = round_page(mmap_size);
956 paddr_t pa, startpa, endpa;
957 vaddr_t va;
958
959 startpa = trunc_page(mmap_start);
960 endpa = round_page(mmap_start + mmap_size);
961 for (pa = startpa, va = vstart; pa < endpa;
962 pa += PAGE_SIZE, va += PAGE_SIZE)
963 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
964 pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
965 memcpy((void *)pa, (caddr_t)vstart + (mmap_start - startpa),
966 mmap_size); /* copy to physical */
967 pmap_kremove(vstart, endpa - startpa);
968
969 for (va = vstart, csize = size; csize > 0;
970 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
971 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
972
973 mmap = (void *)vstart;
974 vstart += size;
975 }
976
977 /* No more KVA stealing after this point. */
978 virtual_avail = vstart;
979
980 /* Now we can reinit the FDT, using the virtual address. */
981 if (fdt)
982 fdt_init(fdt);
983
984 map_func_save = arm64_bs_tag._space_map;
985 arm64_bs_tag._space_map = pmap_bootstrap_bs_map;
986
987 consinit();
988
989 arm64_bs_tag._space_map = map_func_save;
990
991 pmap_avail_fixup();
992
993 uvmexp.pagesize = PAGE_SIZE;
994 uvm_setpagesize();
995
996 /* Make what's left of the initial 64MB block available to UVM. */
997 pmap_physload_avail();
998
999 /* Make all other physical memory available to UVM. */
1000 if (mmap && mmap_desc_ver == EFI_MEMORY_DESCRIPTOR_VERSION) {
1001 EFI_MEMORY_DESCRIPTOR *desc = mmap;
1002
1003 /*
1004 * Load all memory marked as EfiConventionalMemory,
1005 * EfiBootServicesCode or EfiBootServicesData.
1006 * The initial 64MB memory block should be marked as
1007 * EfiLoaderData so it won't be added here.
1008 */
1009 for (i = 0; i < mmap_size / mmap_desc_size; i++) {
1010 #ifdef MMAP_DEBUG
1011 printf("type 0x%x pa 0x%llx va 0x%llx pages 0x%llx attr 0x%llx\n",
1012 desc->Type, desc->PhysicalStart,
1013 desc->VirtualStart, desc->NumberOfPages,
1014 desc->Attribute);
1015 #endif
1016 if (desc->Type == EfiConventionalMemory ||
1017 desc->Type == EfiBootServicesCode ||
1018 desc->Type == EfiBootServicesData) {
1019 reg.addr = desc->PhysicalStart;
1020 reg.size = ptoa(desc->NumberOfPages);
1021 memreg_add(®);
1022 }
1023 desc = NextMemoryDescriptor(desc, mmap_desc_size);
1024 }
1025 } else {
1026 node = fdt_find_node("/memory");
1027 if (node == NULL)
1028 panic("%s: no memory specified", __func__);
1029
1030 for (i = 0; nmemreg < nitems(memreg); i++) {
1031 if (fdt_get_reg(node, i, ®))
1032 break;
1033 if (reg.size == 0)
1034 continue;
1035 memreg_add(®);
1036 }
1037 }
1038
1039 /* Remove reserved memory. */
1040 node = fdt_find_node("/reserved-memory");
1041 if (node) {
1042 for (node = fdt_child_node(node); node;
1043 node = fdt_next_node(node)) {
1044 char *no_map;
1045 if (fdt_node_property(node, "no-map", &no_map) < 0)
1046 continue;
1047 if (fdt_get_reg(node, 0, ®))
1048 continue;
1049 if (reg.size == 0)
1050 continue;
1051 memreg_remove(®);
1052 }
1053 }
1054
1055 /* Remove the initial 64MB block. */
1056 reg.addr = memstart;
1057 reg.size = memend - memstart;
1058 memreg_remove(®);
1059
1060 for (i = 0; i < nmemreg; i++) {
1061 paddr_t start = memreg[i].addr;
1062 paddr_t end = start + memreg[i].size;
1063
1064 uvm_page_physload(atop(start), atop(end),
1065 atop(start), atop(end), 0);
1066 physmem += atop(end - start);
1067 }
1068
1069 kmeminit_nkmempages();
1070
1071 /*
1072 * Make sure that we have enough KVA to initialize UVM. In
1073 * particular, we need enough KVA to be able to allocate the
1074 * vm_page structures and nkmempages for malloc(9).
1075 */
1076 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 1024 * 1024 * 1024 +
1077 physmem * sizeof(struct vm_page) + ptoa(nkmempages));
1078
1079 #ifdef DDB
1080 db_machine_init();
1081
1082 /* Firmware doesn't load symbols. */
1083 ddb_init();
1084
1085 if (boothowto & RB_KDB)
1086 db_enter();
1087 #endif
1088
1089 softintr_init();
1090 splraise(IPL_IPI);
1091 }
1092
1093 char bootargs[256];
1094
1095 void
collect_kernel_args(const char * args)1096 collect_kernel_args(const char *args)
1097 {
1098 /* Make a local copy of the bootargs */
1099 strlcpy(bootargs, args, sizeof(bootargs));
1100 }
1101
1102 void
process_kernel_args(void)1103 process_kernel_args(void)
1104 {
1105 char *cp = bootargs;
1106
1107 if (*cp == 0)
1108 return;
1109
1110 /* Skip the kernel image filename */
1111 while (*cp != ' ' && *cp != 0)
1112 cp++;
1113
1114 if (*cp != 0)
1115 *cp++ = 0;
1116
1117 while (*cp == ' ')
1118 cp++;
1119
1120 boot_args = cp;
1121
1122 printf("bootargs: %s\n", boot_args);
1123
1124 /* Setup pointer to boot flags */
1125 while (*cp != '-')
1126 if (*cp++ == '\0')
1127 return;
1128
1129 while (*cp != 0) {
1130 switch (*cp) {
1131 case 'a':
1132 boothowto |= RB_ASKNAME;
1133 break;
1134 case 'c':
1135 boothowto |= RB_CONFIG;
1136 break;
1137 case 'd':
1138 boothowto |= RB_KDB;
1139 break;
1140 case 's':
1141 boothowto |= RB_SINGLE;
1142 break;
1143 default:
1144 printf("unknown option `%c'\n", *cp);
1145 break;
1146 }
1147 cp++;
1148 }
1149 }
1150
1151 /*
1152 * Allow bootstrap to steal KVA after machdep has given it back to pmap.
1153 */
1154 int
pmap_bootstrap_bs_map(bus_space_tag_t t,bus_addr_t bpa,bus_size_t size,int flags,bus_space_handle_t * bshp)1155 pmap_bootstrap_bs_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
1156 int flags, bus_space_handle_t *bshp)
1157 {
1158 u_long startpa, pa, endpa;
1159 vaddr_t va;
1160 int cache = PMAP_CACHE_DEV_NGNRNE;
1161
1162 if (flags & BUS_SPACE_MAP_PREFETCHABLE)
1163 cache = PMAP_CACHE_CI;
1164
1165 va = virtual_avail; /* steal memory from virtual avail. */
1166
1167 startpa = trunc_page(bpa);
1168 endpa = round_page((bpa + size));
1169
1170 *bshp = (bus_space_handle_t)(va + (bpa - startpa));
1171
1172 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
1173 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, cache);
1174
1175 virtual_avail = va;
1176
1177 return 0;
1178 }
1179
1180 void
memreg_add(const struct fdt_reg * reg)1181 memreg_add(const struct fdt_reg *reg)
1182 {
1183 int i;
1184
1185 for (i = 0; i < nmemreg; i++) {
1186 if (reg->addr == memreg[i].addr + memreg[i].size) {
1187 memreg[i].size += reg->size;
1188 return;
1189 }
1190 if (reg->addr + reg->size == memreg[i].addr) {
1191 memreg[i].addr = reg->addr;
1192 memreg[i].size += reg->size;
1193 return;
1194 }
1195 }
1196
1197 if (nmemreg >= nitems(memreg))
1198 return;
1199
1200 memreg[nmemreg++] = *reg;
1201 }
1202
1203 void
memreg_remove(const struct fdt_reg * reg)1204 memreg_remove(const struct fdt_reg *reg)
1205 {
1206 uint64_t start = reg->addr;
1207 uint64_t end = reg->addr + reg->size;
1208 int i, j;
1209
1210 for (i = 0; i < nmemreg; i++) {
1211 uint64_t memstart = memreg[i].addr;
1212 uint64_t memend = memreg[i].addr + memreg[i].size;
1213
1214 if (end <= memstart)
1215 continue;
1216 if (start >= memend)
1217 continue;
1218
1219 if (start <= memstart)
1220 memstart = MIN(end, memend);
1221 if (end >= memend)
1222 memend = MAX(start, memstart);
1223
1224 if (start > memstart && end < memend) {
1225 if (nmemreg < nitems(memreg)) {
1226 memreg[nmemreg].addr = end;
1227 memreg[nmemreg].size = memend - end;
1228 nmemreg++;
1229 }
1230 memend = start;
1231 }
1232 memreg[i].addr = memstart;
1233 memreg[i].size = memend - memstart;
1234 }
1235
1236 /* Remove empty slots. */
1237 for (i = nmemreg - 1; i >= 0; i--) {
1238 if (memreg[i].size == 0) {
1239 for (j = i; (j + 1) < nmemreg; j++)
1240 memreg[j] = memreg[j + 1];
1241 nmemreg--;
1242 }
1243 }
1244 }
1245