1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2004 Christian Limpach.
5 * Copyright (c) 2004-2006,2008 Kip Macy
6 * Copyright (c) 2008 The NetBSD Foundation, Inc.
7 * Copyright (c) 2013 Roger Pau Monné <roger.pau@citrix.com>
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include "opt_ddb.h"
34 #include "opt_kstack_pages.h"
35
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/kernel.h>
39 #include <sys/reboot.h>
40 #include <sys/systm.h>
41 #include <sys/malloc.h>
42 #include <sys/linker.h>
43 #include <sys/lock.h>
44 #include <sys/rwlock.h>
45 #include <sys/boot.h>
46 #include <sys/ctype.h>
47 #include <sys/mutex.h>
48 #include <sys/smp.h>
49 #include <sys/efi.h>
50 #include <sys/tslog.h>
51
52 #include <vm/vm.h>
53 #include <vm/vm_extern.h>
54 #include <vm/vm_kern.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_object.h>
58 #include <vm/vm_pager.h>
59 #include <vm/vm_param.h>
60
61 #include <machine/_inttypes.h>
62 #include <machine/intr_machdep.h>
63 #include <x86/apicvar.h>
64 #include <x86/init.h>
65 #include <machine/pc/bios.h>
66 #include <machine/smp.h>
67 #include <machine/intr_machdep.h>
68 #include <machine/md_var.h>
69 #include <machine/metadata.h>
70 #include <machine/cpu.h>
71
72 #include <xen/xen-os.h>
73 #include <xen/hvm.h>
74 #include <xen/hypervisor.h>
75 #include <xen/xenstore/xenstorevar.h>
76 #include <xen/xen_pv.h>
77
78 #include <contrib/xen/arch-x86/cpuid.h>
79 #include <contrib/xen/arch-x86/hvm/start_info.h>
80 #include <contrib/xen/vcpu.h>
81
82 #include <dev/xen/timer/timer.h>
83
84 #ifdef DDB
85 #include <ddb/ddb.h>
86 #endif
87
88 /* Native initial function */
89 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
90 /* Xen initial function */
91 uint64_t hammer_time_xen(vm_paddr_t);
92
93 #define MAX_E820_ENTRIES 128
94
95 /*--------------------------- Forward Declarations ---------------------------*/
96 static caddr_t xen_pvh_parse_preload_data(uint64_t);
97 static void pvh_parse_memmap(caddr_t, vm_paddr_t *, int *);
98
99 /*---------------------------- Extern Declarations ---------------------------*/
100 /*
101 * Placed by the linker at the end of the bss section, which is the last
102 * section loaded by Xen before loading the symtab and strtab.
103 */
104 extern uint32_t end;
105
106 /*-------------------------------- Global Data -------------------------------*/
107 struct init_ops xen_pvh_init_ops = {
108 .parse_preload_data = xen_pvh_parse_preload_data,
109 .early_clock_source_init = xen_clock_init,
110 .early_delay = xen_delay,
111 .parse_memmap = pvh_parse_memmap,
112 };
113
114 static struct bios_smap xen_smap[MAX_E820_ENTRIES];
115
116 static struct hvm_start_info *start_info;
117
118 /*-------------------------------- Xen PV init -------------------------------*/
119
120 static int
isxen(void)121 isxen(void)
122 {
123 static int xen = -1;
124 uint32_t base;
125 u_int regs[4];
126
127 if (xen != -1)
128 return (xen);
129
130 /*
131 * The full code for identifying which hypervisor we're running under
132 * is in sys/x86/x86/identcpu.c and runs later in the boot process;
133 * this is sufficient to distinguish Xen PVH booting from non-Xen PVH
134 * and skip some very early Xen-specific code in the non-Xen case.
135 */
136 xen = 0;
137 for (base = 0x40000000; base < 0x40010000; base += 0x100) {
138 do_cpuid(base, regs);
139 if (regs[1] == XEN_CPUID_SIGNATURE_EBX &&
140 regs[2] == XEN_CPUID_SIGNATURE_ECX &&
141 regs[3] == XEN_CPUID_SIGNATURE_EDX) {
142 xen = 1;
143 break;
144 }
145 }
146 return (xen);
147 }
148
149 #define CRASH(...) do { \
150 if (isxen()) { \
151 xc_printf(__VA_ARGS__); \
152 HYPERVISOR_shutdown(SHUTDOWN_crash); \
153 } else { \
154 halt(); \
155 } \
156 } while (0)
157
158 uint64_t
hammer_time_xen(vm_paddr_t start_info_paddr)159 hammer_time_xen(vm_paddr_t start_info_paddr)
160 {
161 struct hvm_modlist_entry *mod;
162 struct xen_add_to_physmap xatp;
163 uint64_t physfree;
164 char *kenv;
165 int rc;
166
167 if (isxen()) {
168 xen_domain_type = XEN_HVM_DOMAIN;
169 vm_guest = VM_GUEST_XEN;
170 rc = xen_hvm_init_hypercall_stubs(XEN_HVM_INIT_EARLY);
171 if (rc) {
172 xc_printf("ERROR: failed to initialize hypercall page: %d\n",
173 rc);
174 HYPERVISOR_shutdown(SHUTDOWN_crash);
175 }
176 }
177
178 start_info = (struct hvm_start_info *)(start_info_paddr + KERNBASE);
179 if (start_info->magic != XEN_HVM_START_MAGIC_VALUE) {
180 CRASH("Unknown magic value in start_info struct: %#x\n",
181 start_info->magic);
182 }
183
184 /*
185 * Select the higher address to use as physfree: either after
186 * start_info, after the kernel, after the memory map or after any of
187 * the modules. We assume enough memory to be available after the
188 * selected address for the needs of very early memory allocations.
189 */
190 physfree = roundup2(start_info_paddr + sizeof(struct hvm_start_info),
191 PAGE_SIZE);
192 physfree = MAX(roundup2((vm_paddr_t)_end - KERNBASE, PAGE_SIZE),
193 physfree);
194
195 if (start_info->memmap_paddr != 0)
196 physfree = MAX(roundup2(start_info->memmap_paddr +
197 start_info->memmap_entries *
198 sizeof(struct hvm_memmap_table_entry), PAGE_SIZE),
199 physfree);
200
201 if (start_info->modlist_paddr != 0) {
202 unsigned int i;
203
204 if (start_info->nr_modules == 0) {
205 CRASH(
206 "ERROR: modlist_paddr != 0 but nr_modules == 0\n");
207 }
208 mod = (struct hvm_modlist_entry *)
209 (start_info->modlist_paddr + KERNBASE);
210 for (i = 0; i < start_info->nr_modules; i++)
211 physfree = MAX(roundup2(mod[i].paddr + mod[i].size,
212 PAGE_SIZE), physfree);
213 }
214
215 if (isxen()) {
216 xatp.domid = DOMID_SELF;
217 xatp.idx = 0;
218 xatp.space = XENMAPSPACE_shared_info;
219 xatp.gpfn = atop(physfree);
220 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) {
221 xc_printf("ERROR: failed to setup shared_info page\n");
222 HYPERVISOR_shutdown(SHUTDOWN_crash);
223 }
224 HYPERVISOR_shared_info = (shared_info_t *)(physfree + KERNBASE);
225 physfree += PAGE_SIZE;
226 }
227
228 /*
229 * Init a static kenv using a free page. The contents will be filled
230 * from the parse_preload_data hook.
231 */
232 kenv = (void *)(physfree + KERNBASE);
233 physfree += PAGE_SIZE;
234 bzero_early(kenv, PAGE_SIZE);
235 init_static_kenv(kenv, PAGE_SIZE);
236
237 /* Set the hooks for early functions that diverge from bare metal */
238 init_ops = xen_pvh_init_ops;
239 hvm_start_flags = start_info->flags;
240
241 /* Now we can jump into the native init function */
242 return (hammer_time(0, physfree));
243 }
244
245 /*-------------------------------- PV specific -------------------------------*/
246
247 /*
248 * When booted as a PVH guest FreeBSD needs to avoid using the RSDP address
249 * hint provided by the loader because it points to the native set of ACPI
250 * tables instead of the ones crafted by Xen. The acpi.rsdp env variable is
251 * removed from kenv if present, and a new acpi.rsdp is added to kenv that
252 * points to the address of the Xen crafted RSDP.
253 */
reject_option(const char * option)254 static bool reject_option(const char *option)
255 {
256 static const char *reject[] = {
257 "acpi.rsdp",
258 };
259 unsigned int i;
260
261 for (i = 0; i < nitems(reject); i++)
262 if (strncmp(option, reject[i], strlen(reject[i])) == 0)
263 return (true);
264
265 return (false);
266 }
267
268 static void
xen_pvh_set_env(char * env,bool (* filter)(const char *))269 xen_pvh_set_env(char *env, bool (*filter)(const char *))
270 {
271 char *option;
272
273 if (env == NULL)
274 return;
275
276 option = env;
277 while (*option != 0) {
278 char *value;
279
280 if (filter != NULL && filter(option)) {
281 option += strlen(option) + 1;
282 continue;
283 }
284
285 value = option;
286 option = strsep(&value, "=");
287 if (kern_setenv(option, value) != 0 && isxen())
288 xc_printf("unable to add kenv %s=%s\n", option, value);
289 option = value + strlen(value) + 1;
290 }
291 }
292
293 #ifdef DDB
294 /*
295 * The way Xen loads the symtab is different from the native boot loader,
296 * because it's tailored for NetBSD. So we have to adapt and use the same
297 * method as NetBSD. Portions of the code below have been picked from NetBSD:
298 * sys/kern/kern_ksyms.c CVS Revision 1.71.
299 */
300 static void
xen_pvh_parse_symtab(void)301 xen_pvh_parse_symtab(void)
302 {
303 Elf_Ehdr *ehdr;
304 Elf_Shdr *shdr;
305 int i, j;
306
307 ehdr = (Elf_Ehdr *)(&end + 1);
308 if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) ||
309 ehdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
310 ehdr->e_version > 1) {
311 if (isxen())
312 xc_printf("Unable to load ELF symtab: invalid symbol table\n");
313 return;
314 }
315
316 shdr = (Elf_Shdr *)((uint8_t *)ehdr + ehdr->e_shoff);
317 /* Find the symbol table and the corresponding string table. */
318 for (i = 1; i < ehdr->e_shnum; i++) {
319 if (shdr[i].sh_type != SHT_SYMTAB)
320 continue;
321 if (shdr[i].sh_offset == 0)
322 continue;
323 ksymtab = (uintptr_t)((uint8_t *)ehdr + shdr[i].sh_offset);
324 ksymtab_size = shdr[i].sh_size;
325 j = shdr[i].sh_link;
326 if (shdr[j].sh_offset == 0)
327 continue; /* Can this happen? */
328 kstrtab = (uintptr_t)((uint8_t *)ehdr + shdr[j].sh_offset);
329 break;
330 }
331
332 if ((ksymtab == 0 || kstrtab == 0) && isxen())
333 xc_printf(
334 "Unable to load ELF symtab: could not find symtab or strtab\n");
335 }
336 #endif
337
338 static void
fixup_console(caddr_t kmdp)339 fixup_console(caddr_t kmdp)
340 {
341 struct xen_platform_op op = {
342 .cmd = XENPF_get_dom0_console,
343 };
344 xenpf_dom0_console_t *console = &op.u.dom0_console;
345 union {
346 struct efi_fb efi;
347 struct vbe_fb vbe;
348 } *fb = NULL;
349 int size;
350
351 size = HYPERVISOR_platform_op(&op);
352 if (size < 0) {
353 xc_printf("Failed to get dom0 video console info: %d\n", size);
354 return;
355 }
356
357 switch (console->video_type) {
358 case XEN_VGATYPE_VESA_LFB:
359 fb = (__typeof__ (fb))preload_search_info(kmdp,
360 MODINFO_METADATA | MODINFOMD_VBE_FB);
361
362 if (fb == NULL) {
363 xc_printf("No VBE FB in kernel metadata\n");
364 return;
365 }
366
367 _Static_assert(offsetof(struct vbe_fb, fb_bpp) ==
368 offsetof(struct efi_fb, fb_mask_reserved) +
369 sizeof(fb->efi.fb_mask_reserved),
370 "Bad structure overlay\n");
371 fb->vbe.fb_bpp = console->u.vesa_lfb.bits_per_pixel;
372 /* FALLTHROUGH */
373 case XEN_VGATYPE_EFI_LFB:
374 if (fb == NULL) {
375 fb = (__typeof__ (fb))preload_search_info(kmdp,
376 MODINFO_METADATA | MODINFOMD_EFI_FB);
377 if (fb == NULL) {
378 xc_printf("No EFI FB in kernel metadata\n");
379 return;
380 }
381 }
382
383 fb->efi.fb_addr = console->u.vesa_lfb.lfb_base;
384 if (size >
385 offsetof(xenpf_dom0_console_t, u.vesa_lfb.ext_lfb_base))
386 fb->efi.fb_addr |=
387 (uint64_t)console->u.vesa_lfb.ext_lfb_base << 32;
388 fb->efi.fb_size = console->u.vesa_lfb.lfb_size << 16;
389 fb->efi.fb_height = console->u.vesa_lfb.height;
390 fb->efi.fb_width = console->u.vesa_lfb.width;
391 fb->efi.fb_stride = (console->u.vesa_lfb.bytes_per_line << 3) /
392 console->u.vesa_lfb.bits_per_pixel;
393 #define FBMASK(c) \
394 ((~0u << console->u.vesa_lfb.c ## _pos) & \
395 (~0u >> (32 - console->u.vesa_lfb.c ## _pos - \
396 console->u.vesa_lfb.c ## _size)))
397 fb->efi.fb_mask_red = FBMASK(red);
398 fb->efi.fb_mask_green = FBMASK(green);
399 fb->efi.fb_mask_blue = FBMASK(blue);
400 fb->efi.fb_mask_reserved = FBMASK(rsvd);
401 #undef FBMASK
402 break;
403
404 default:
405 xc_printf("Video console type unsupported\n");
406 return;
407 }
408 }
409
410 static caddr_t
xen_pvh_parse_preload_data(uint64_t modulep)411 xen_pvh_parse_preload_data(uint64_t modulep)
412 {
413 caddr_t kmdp;
414 vm_ooffset_t off;
415 vm_paddr_t metadata;
416 char *envp;
417 char acpi_rsdp[19];
418
419 TSENTER();
420 if (start_info->modlist_paddr != 0) {
421 struct hvm_modlist_entry *mod;
422 const char *cmdline;
423
424 mod = (struct hvm_modlist_entry *)
425 (start_info->modlist_paddr + KERNBASE);
426 cmdline = mod[0].cmdline_paddr ?
427 (const char *)(mod[0].cmdline_paddr + KERNBASE) : NULL;
428
429 if (strcmp(cmdline, "header") == 0) {
430 struct xen_header *header;
431
432 header = (struct xen_header *)(mod[0].paddr + KERNBASE);
433
434 if ((header->flags & XENHEADER_HAS_MODULEP_OFFSET) !=
435 XENHEADER_HAS_MODULEP_OFFSET) {
436 xc_printf("Unable to load module metadata\n");
437 HYPERVISOR_shutdown(SHUTDOWN_crash);
438 }
439
440 preload_metadata = (caddr_t)(mod[0].paddr +
441 header->modulep_offset + KERNBASE);
442
443 kmdp = preload_search_by_type("elf kernel");
444 if (kmdp == NULL)
445 kmdp = preload_search_by_type("elf64 kernel");
446 if (kmdp == NULL) {
447 xc_printf("Unable to find kernel\n");
448 HYPERVISOR_shutdown(SHUTDOWN_crash);
449 }
450
451 /*
452 * Xen has relocated the metadata and the modules, so
453 * we need to recalculate it's position. This is done
454 * by saving the original modulep address and then
455 * calculating the offset from the real modulep
456 * position.
457 */
458 metadata = MD_FETCH(kmdp, MODINFOMD_MODULEP,
459 vm_paddr_t);
460 off = mod[0].paddr + header->modulep_offset - metadata +
461 KERNBASE;
462 } else {
463 preload_metadata = (caddr_t)(mod[0].paddr + KERNBASE);
464
465 kmdp = preload_search_by_type("elf kernel");
466 if (kmdp == NULL)
467 kmdp = preload_search_by_type("elf64 kernel");
468 if (kmdp == NULL) {
469 xc_printf("Unable to find kernel\n");
470 HYPERVISOR_shutdown(SHUTDOWN_crash);
471 }
472
473 metadata = MD_FETCH(kmdp, MODINFOMD_MODULEP, vm_paddr_t);
474 off = mod[0].paddr + KERNBASE - metadata;
475 }
476
477 preload_bootstrap_relocate(off);
478
479 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int);
480 envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *);
481 if (envp != NULL)
482 envp += off;
483 xen_pvh_set_env(envp, reject_option);
484
485 if (MD_FETCH(kmdp, MODINFOMD_EFI_MAP, void *) != NULL)
486 strlcpy(bootmethod, "UEFI", sizeof(bootmethod));
487 else
488 strlcpy(bootmethod, "BIOS", sizeof(bootmethod));
489
490 fixup_console(kmdp);
491 } else {
492 /* Parse the extra boot information given by Xen */
493 if (start_info->cmdline_paddr != 0)
494 boot_parse_cmdline_delim(
495 (char *)(start_info->cmdline_paddr + KERNBASE),
496 ", \t\n");
497 kmdp = NULL;
498 strlcpy(bootmethod, "PVH", sizeof(bootmethod));
499 }
500
501 boothowto |= boot_env_to_howto();
502
503 snprintf(acpi_rsdp, sizeof(acpi_rsdp), "%#" PRIx64,
504 start_info->rsdp_paddr);
505 kern_setenv("acpi.rsdp", acpi_rsdp);
506
507 #ifdef DDB
508 xen_pvh_parse_symtab();
509 #endif
510 TSEXIT();
511 return (kmdp);
512 }
513
514 static void
pvh_parse_memmap_start_info(caddr_t kmdp,vm_paddr_t * physmap,int * physmap_idx)515 pvh_parse_memmap_start_info(caddr_t kmdp, vm_paddr_t *physmap,
516 int *physmap_idx)
517 {
518 const struct hvm_memmap_table_entry * entries;
519 size_t nentries;
520 size_t i;
521
522 /* Extract from HVM start_info. */
523 entries = (struct hvm_memmap_table_entry *)(start_info->memmap_paddr + KERNBASE);
524 nentries = start_info->memmap_entries;
525
526 /* Convert into E820 format and handle one by one. */
527 for (i = 0; i < nentries; i++) {
528 struct bios_smap entry;
529
530 entry.base = entries[i].addr;
531 entry.length = entries[i].size;
532
533 /*
534 * Luckily for us, the XEN_HVM_MEMMAP_TYPE_* values exactly
535 * match the SMAP_TYPE_* values so we don't need to translate
536 * anything here.
537 */
538 entry.type = entries[i].type;
539
540 bios_add_smap_entries(&entry, 1, physmap, physmap_idx);
541 }
542 }
543
544 static void
xen_pvh_parse_memmap(caddr_t kmdp,vm_paddr_t * physmap,int * physmap_idx)545 xen_pvh_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
546 {
547 struct xen_memory_map memmap;
548 u_int32_t size;
549 int rc;
550
551 /* We should only reach here if we're running under Xen. */
552 KASSERT(isxen(), ("xen_pvh_parse_memmap reached when !Xen"));
553
554 /* Fetch the E820 map from Xen */
555 memmap.nr_entries = MAX_E820_ENTRIES;
556 set_xen_guest_handle(memmap.buffer, xen_smap);
557 rc = HYPERVISOR_memory_op(XENMEM_memory_map, &memmap);
558 if (rc) {
559 xc_printf("ERROR: unable to fetch Xen E820 memory map: %d\n",
560 rc);
561 HYPERVISOR_shutdown(SHUTDOWN_crash);
562 }
563
564 size = memmap.nr_entries * sizeof(xen_smap[0]);
565
566 bios_add_smap_entries(xen_smap, size, physmap, physmap_idx);
567 }
568
569 static void
pvh_parse_memmap(caddr_t kmdp,vm_paddr_t * physmap,int * physmap_idx)570 pvh_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
571 {
572
573 /*
574 * If version >= 1 and memmap_paddr != 0, use the memory map provided
575 * in the start_info structure; if not, we're running under legacy
576 * Xen and need to use the Xen hypercall.
577 */
578 if ((start_info->version >= 1) && (start_info->memmap_paddr != 0))
579 pvh_parse_memmap_start_info(kmdp, physmap, physmap_idx);
580 else
581 xen_pvh_parse_memmap(kmdp, physmap, physmap_idx);
582 }
583