1 /* $NetBSD: subr_kobj.c,v 1.78 2023/04/28 07:33:57 skrll Exp $ */
2
3 /*
4 * Copyright (c) 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software developed for The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 1998-2000 Doug Rabson
34 * Copyright (c) 2004 Peter Wemm
35 * All rights reserved.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 */
58
59 /*
60 * Kernel loader for ELF objects.
61 *
62 * TODO: adjust kmem_alloc() calls to avoid needless fragmentation.
63 */
64
65 #include <sys/cdefs.h>
66 __KERNEL_RCSID(0, "$NetBSD: subr_kobj.c,v 1.78 2023/04/28 07:33:57 skrll Exp $");
67
68 #ifdef _KERNEL_OPT
69 #include "opt_modular.h"
70 #endif
71
72 #include <sys/kobj_impl.h>
73
74 #ifdef MODULAR
75
76 #include <sys/param.h>
77
78 #include <sys/kernel.h>
79 #include <sys/kmem.h>
80 #include <sys/ksyms.h>
81 #include <sys/module.h>
82 #include <sys/proc.h>
83
84 #include <uvm/uvm_extern.h>
85
86 #define kobj_error(_kobj, ...) \
87 kobj_out(__func__, __LINE__, _kobj, __VA_ARGS__)
88
89 static int kobj_relocate(kobj_t, bool);
90 static int kobj_checksyms(kobj_t, bool);
91 static void kobj_out(const char *, int, kobj_t, const char *, ...)
92 __printflike(4, 5);
93 static void kobj_jettison(kobj_t);
94 static void kobj_free(kobj_t, void *, size_t);
95 static void kobj_close(kobj_t);
96 static int kobj_read_mem(kobj_t, void **, size_t, off_t, bool);
97 static void kobj_close_mem(kobj_t);
98
99 /*
100 * kobj_load_mem:
101 *
102 * Load an object already resident in memory. If size is not -1,
103 * the complete size of the object is known.
104 */
105 int
kobj_load_mem(kobj_t * kop,const char * name,void * base,ssize_t size)106 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size)
107 {
108 kobj_t ko;
109
110 ko = kmem_zalloc(sizeof(*ko), KM_SLEEP);
111 ko->ko_type = KT_MEMORY;
112 kobj_setname(ko, name);
113 ko->ko_source = base;
114 ko->ko_memsize = size;
115 ko->ko_read = kobj_read_mem;
116 ko->ko_close = kobj_close_mem;
117
118 *kop = ko;
119 return kobj_load(ko);
120 }
121
122 /*
123 * kobj_close:
124 *
125 * Close an open ELF object.
126 */
127 static void
kobj_close(kobj_t ko)128 kobj_close(kobj_t ko)
129 {
130
131 if (ko->ko_source == NULL) {
132 return;
133 }
134
135 ko->ko_close(ko);
136 ko->ko_source = NULL;
137 }
138
139 static void
kobj_close_mem(kobj_t ko)140 kobj_close_mem(kobj_t ko)
141 {
142
143 return;
144 }
145
146 /*
147 * kobj_load:
148 *
149 * Load an ELF object and prepare to link into the running kernel
150 * image.
151 */
152 int
kobj_load(kobj_t ko)153 kobj_load(kobj_t ko)
154 {
155 Elf_Ehdr *hdr;
156 Elf_Shdr *shdr;
157 Elf_Sym *es;
158 vaddr_t map_text_base;
159 vaddr_t map_data_base;
160 vaddr_t map_rodata_base;
161 size_t map_text_size;
162 size_t map_data_size;
163 size_t map_rodata_size;
164 int error;
165 int symtabindex;
166 int symstrindex;
167 int nsym;
168 int pb, rl, ra;
169 int alignmask;
170 int i, j;
171 void *addr;
172
173 KASSERT(ko->ko_type != KT_UNSET);
174 KASSERT(ko->ko_source != NULL);
175
176 shdr = NULL;
177 error = 0;
178 hdr = NULL;
179
180 /*
181 * Read the elf header from the file.
182 */
183 error = ko->ko_read(ko, (void **)&hdr, sizeof(*hdr), 0, true);
184 if (error != 0) {
185 kobj_error(ko, "read failed %d", error);
186 goto out;
187 }
188 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) {
189 kobj_error(ko, "not an ELF object");
190 error = ENOEXEC;
191 goto out;
192 }
193
194 if (hdr->e_ident[EI_VERSION] != EV_CURRENT ||
195 hdr->e_version != EV_CURRENT) {
196 kobj_error(ko, "unsupported file version %d",
197 hdr->e_ident[EI_VERSION]);
198 error = ENOEXEC;
199 goto out;
200 }
201 if (hdr->e_type != ET_REL) {
202 kobj_error(ko, "unsupported file type %d", hdr->e_type);
203 error = ENOEXEC;
204 goto out;
205 }
206 switch (hdr->e_machine) {
207 #if ELFSIZE == 32
208 ELF32_MACHDEP_ID_CASES
209 #elif ELFSIZE == 64
210 ELF64_MACHDEP_ID_CASES
211 #else
212 #error not defined
213 #endif
214 default:
215 kobj_error(ko, "unsupported machine %d", hdr->e_machine);
216 error = ENOEXEC;
217 goto out;
218 }
219
220 ko->ko_nprogtab = 0;
221 ko->ko_shdr = 0;
222 ko->ko_nrel = 0;
223 ko->ko_nrela = 0;
224
225 /*
226 * Allocate and read in the section header.
227 */
228 if (hdr->e_shnum == 0 || hdr->e_shnum > ELF_MAXSHNUM ||
229 hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) {
230 kobj_error(ko, "bad sizes");
231 error = ENOEXEC;
232 goto out;
233 }
234 ko->ko_shdrsz = hdr->e_shnum * sizeof(Elf_Shdr);
235 error = ko->ko_read(ko, (void **)&shdr, ko->ko_shdrsz, hdr->e_shoff,
236 true);
237 if (error != 0) {
238 kobj_error(ko, "read failed %d", error);
239 goto out;
240 }
241 ko->ko_shdr = shdr;
242
243 /*
244 * Scan the section header for information and table sizing.
245 */
246 nsym = 0;
247 symtabindex = symstrindex = -1;
248 for (i = 0; i < hdr->e_shnum; i++) {
249 switch (shdr[i].sh_type) {
250 case SHT_PROGBITS:
251 case SHT_NOBITS:
252 ko->ko_nprogtab++;
253 break;
254 case SHT_SYMTAB:
255 nsym++;
256 symtabindex = i;
257 symstrindex = shdr[i].sh_link;
258 break;
259 case SHT_REL:
260 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS)
261 continue;
262 ko->ko_nrel++;
263 break;
264 case SHT_RELA:
265 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS)
266 continue;
267 ko->ko_nrela++;
268 break;
269 case SHT_STRTAB:
270 break;
271 }
272 }
273 if (ko->ko_nprogtab == 0) {
274 kobj_error(ko, "file has no contents");
275 error = ENOEXEC;
276 goto out;
277 }
278 if (nsym != 1) {
279 /* Only allow one symbol table for now */
280 kobj_error(ko, "file has no valid symbol table");
281 error = ENOEXEC;
282 goto out;
283 }
284 KASSERT(symtabindex != -1);
285 KASSERT(symstrindex != -1);
286
287 if (symstrindex == SHN_UNDEF || symstrindex >= hdr->e_shnum ||
288 shdr[symstrindex].sh_type != SHT_STRTAB) {
289 kobj_error(ko, "file has invalid symbol strings");
290 error = ENOEXEC;
291 goto out;
292 }
293
294 /*
295 * Allocate space for tracking the load chunks.
296 */
297 if (ko->ko_nprogtab != 0) {
298 ko->ko_progtab = kmem_zalloc(ko->ko_nprogtab *
299 sizeof(*ko->ko_progtab), KM_SLEEP);
300 if (ko->ko_progtab == NULL) {
301 error = ENOMEM;
302 kobj_error(ko, "out of memory");
303 goto out;
304 }
305 }
306 if (ko->ko_nrel != 0) {
307 ko->ko_reltab = kmem_zalloc(ko->ko_nrel *
308 sizeof(*ko->ko_reltab), KM_SLEEP);
309 if (ko->ko_reltab == NULL) {
310 error = ENOMEM;
311 kobj_error(ko, "out of memory");
312 goto out;
313 }
314 }
315 if (ko->ko_nrela != 0) {
316 ko->ko_relatab = kmem_zalloc(ko->ko_nrela *
317 sizeof(*ko->ko_relatab), KM_SLEEP);
318 if (ko->ko_relatab == NULL) {
319 error = ENOMEM;
320 kobj_error(ko, "out of memory");
321 goto out;
322 }
323 }
324
325 /*
326 * Allocate space for and load the symbol table.
327 */
328 ko->ko_symcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
329 if (ko->ko_symcnt == 0) {
330 kobj_error(ko, "no symbol table");
331 error = ENOEXEC;
332 goto out;
333 }
334 error = ko->ko_read(ko, (void **)&ko->ko_symtab,
335 ko->ko_symcnt * sizeof(Elf_Sym),
336 shdr[symtabindex].sh_offset, true);
337 if (error != 0) {
338 kobj_error(ko, "read failed %d", error);
339 goto out;
340 }
341
342 /*
343 * Allocate space for and load the symbol strings.
344 */
345 ko->ko_strtabsz = shdr[symstrindex].sh_size;
346 if (ko->ko_strtabsz == 0) {
347 kobj_error(ko, "no symbol strings");
348 error = ENOEXEC;
349 goto out;
350 }
351 error = ko->ko_read(ko, (void *)&ko->ko_strtab, ko->ko_strtabsz,
352 shdr[symstrindex].sh_offset, true);
353 if (error != 0) {
354 kobj_error(ko, "read failed %d", error);
355 goto out;
356 }
357
358 /*
359 * Adjust module symbol namespace, if necessary (e.g. with rump)
360 */
361 error = kobj_renamespace(ko->ko_symtab, ko->ko_symcnt,
362 &ko->ko_strtab, &ko->ko_strtabsz);
363 if (error != 0) {
364 kobj_error(ko, "renamespace failed %d", error);
365 goto out;
366 }
367
368 /*
369 * Do we have a string table for the section names?
370 */
371 if (hdr->e_shstrndx != SHN_UNDEF) {
372 if (hdr->e_shstrndx >= hdr->e_shnum) {
373 kobj_error(ko, "bad shstrndx");
374 error = ENOEXEC;
375 goto out;
376 }
377 if (shdr[hdr->e_shstrndx].sh_size != 0 &&
378 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
379 ko->ko_shstrtabsz = shdr[hdr->e_shstrndx].sh_size;
380 error = ko->ko_read(ko, (void **)&ko->ko_shstrtab,
381 shdr[hdr->e_shstrndx].sh_size,
382 shdr[hdr->e_shstrndx].sh_offset, true);
383 if (error != 0) {
384 kobj_error(ko, "read failed %d", error);
385 goto out;
386 }
387 }
388 }
389
390 /*
391 * Size up code/data(progbits) and bss(nobits).
392 */
393 alignmask = 0;
394 map_text_size = 0;
395 map_data_size = 0;
396 map_rodata_size = 0;
397 for (i = 0; i < hdr->e_shnum; i++) {
398 if (shdr[i].sh_type != SHT_PROGBITS &&
399 shdr[i].sh_type != SHT_NOBITS)
400 continue;
401 alignmask = shdr[i].sh_addralign - 1;
402 if ((shdr[i].sh_flags & SHF_EXECINSTR)) {
403 map_text_size += alignmask;
404 map_text_size &= ~alignmask;
405 map_text_size += shdr[i].sh_size;
406 } else if (!(shdr[i].sh_flags & SHF_WRITE)) {
407 map_rodata_size += alignmask;
408 map_rodata_size &= ~alignmask;
409 map_rodata_size += shdr[i].sh_size;
410 } else {
411 map_data_size += alignmask;
412 map_data_size &= ~alignmask;
413 map_data_size += shdr[i].sh_size;
414 }
415 }
416
417 if (map_text_size == 0) {
418 kobj_error(ko, "no text");
419 error = ENOEXEC;
420 goto out;
421 }
422
423 if (map_data_size != 0) {
424 map_data_base = uvm_km_alloc(module_map, round_page(map_data_size),
425 0, UVM_KMF_WIRED);
426 if (map_data_base == 0) {
427 kobj_error(ko, "out of memory");
428 error = ENOMEM;
429 goto out;
430 }
431 ko->ko_data_address = map_data_base;
432 ko->ko_data_size = map_data_size;
433 } else {
434 map_data_base = 0;
435 ko->ko_data_address = 0;
436 ko->ko_data_size = 0;
437 }
438
439 if (map_rodata_size != 0) {
440 map_rodata_base = uvm_km_alloc(module_map, round_page(map_rodata_size),
441 0, UVM_KMF_WIRED);
442 if (map_rodata_base == 0) {
443 kobj_error(ko, "out of memory");
444 error = ENOMEM;
445 goto out;
446 }
447 ko->ko_rodata_address = map_rodata_base;
448 ko->ko_rodata_size = map_rodata_size;
449 } else {
450 map_rodata_base = 0;
451 ko->ko_rodata_address = 0;
452 ko->ko_rodata_size = 0;
453 }
454
455 map_text_base = uvm_km_alloc(module_map, round_page(map_text_size),
456 0, UVM_KMF_WIRED | UVM_KMF_EXEC);
457 if (map_text_base == 0) {
458 kobj_error(ko, "out of memory");
459 error = ENOMEM;
460 goto out;
461 }
462 ko->ko_text_address = map_text_base;
463 ko->ko_text_size = map_text_size;
464
465 /*
466 * Now load code/data(progbits), zero bss(nobits), allocate space
467 * for and load relocs
468 */
469 pb = 0;
470 rl = 0;
471 ra = 0;
472 alignmask = 0;
473 for (i = 0; i < hdr->e_shnum; i++) {
474 switch (shdr[i].sh_type) {
475 case SHT_PROGBITS:
476 case SHT_NOBITS:
477 alignmask = shdr[i].sh_addralign - 1;
478 if ((shdr[i].sh_flags & SHF_EXECINSTR)) {
479 map_text_base += alignmask;
480 map_text_base &= ~alignmask;
481 addr = (void *)map_text_base;
482 map_text_base += shdr[i].sh_size;
483 } else if (!(shdr[i].sh_flags & SHF_WRITE)) {
484 map_rodata_base += alignmask;
485 map_rodata_base &= ~alignmask;
486 addr = (void *)map_rodata_base;
487 map_rodata_base += shdr[i].sh_size;
488 } else {
489 map_data_base += alignmask;
490 map_data_base &= ~alignmask;
491 addr = (void *)map_data_base;
492 map_data_base += shdr[i].sh_size;
493 }
494
495 ko->ko_progtab[pb].addr = addr;
496 if (shdr[i].sh_type == SHT_PROGBITS) {
497 ko->ko_progtab[pb].name = "<<PROGBITS>>";
498 error = ko->ko_read(ko, &addr,
499 shdr[i].sh_size, shdr[i].sh_offset, false);
500 if (error != 0) {
501 kobj_error(ko, "read failed %d", error);
502 goto out;
503 }
504 } else { /* SHT_NOBITS */
505 ko->ko_progtab[pb].name = "<<NOBITS>>";
506 memset(addr, 0, shdr[i].sh_size);
507 }
508
509 ko->ko_progtab[pb].size = shdr[i].sh_size;
510 ko->ko_progtab[pb].sec = i;
511 if (ko->ko_shstrtab != NULL && shdr[i].sh_name != 0) {
512 ko->ko_progtab[pb].name =
513 ko->ko_shstrtab + shdr[i].sh_name;
514 }
515
516 /* Update all symbol values with the offset. */
517 for (j = 0; j < ko->ko_symcnt; j++) {
518 es = &ko->ko_symtab[j];
519 if (es->st_shndx != i) {
520 continue;
521 }
522 es->st_value += (Elf_Addr)addr;
523 }
524 pb++;
525 break;
526 case SHT_REL:
527 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS)
528 break;
529 ko->ko_reltab[rl].size = shdr[i].sh_size;
530 ko->ko_reltab[rl].size -=
531 shdr[i].sh_size % sizeof(Elf_Rel);
532 if (ko->ko_reltab[rl].size != 0) {
533 ko->ko_reltab[rl].nrel =
534 shdr[i].sh_size / sizeof(Elf_Rel);
535 ko->ko_reltab[rl].sec = shdr[i].sh_info;
536 error = ko->ko_read(ko,
537 (void **)&ko->ko_reltab[rl].rel,
538 ko->ko_reltab[rl].size,
539 shdr[i].sh_offset, true);
540 if (error != 0) {
541 kobj_error(ko, "read failed %d",
542 error);
543 goto out;
544 }
545 }
546 rl++;
547 break;
548 case SHT_RELA:
549 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS)
550 break;
551 ko->ko_relatab[ra].size = shdr[i].sh_size;
552 ko->ko_relatab[ra].size -=
553 shdr[i].sh_size % sizeof(Elf_Rela);
554 if (ko->ko_relatab[ra].size != 0) {
555 ko->ko_relatab[ra].nrela =
556 shdr[i].sh_size / sizeof(Elf_Rela);
557 ko->ko_relatab[ra].sec = shdr[i].sh_info;
558 error = ko->ko_read(ko,
559 (void **)&ko->ko_relatab[ra].rela,
560 shdr[i].sh_size,
561 shdr[i].sh_offset, true);
562 if (error != 0) {
563 kobj_error(ko, "read failed %d", error);
564 goto out;
565 }
566 }
567 ra++;
568 break;
569 default:
570 break;
571 }
572 }
573 if (pb != ko->ko_nprogtab) {
574 panic("%s:%d: %s: lost progbits", __func__, __LINE__,
575 ko->ko_name);
576 }
577 if (rl != ko->ko_nrel) {
578 panic("%s:%d: %s: lost rel", __func__, __LINE__,
579 ko->ko_name);
580 }
581 if (ra != ko->ko_nrela) {
582 panic("%s:%d: %s: lost rela", __func__, __LINE__,
583 ko->ko_name);
584 }
585 if (map_text_base != ko->ko_text_address + map_text_size) {
586 panic("%s:%d: %s: map_text_base 0x%lx != address %lx "
587 "+ map_text_size %ld (0x%lx)\n",
588 __func__, __LINE__, ko->ko_name, (long)map_text_base,
589 (long)ko->ko_text_address, (long)map_text_size,
590 (long)ko->ko_text_address + map_text_size);
591 }
592 if (map_data_base != ko->ko_data_address + map_data_size) {
593 panic("%s:%d: %s: map_data_base 0x%lx != address %lx "
594 "+ map_data_size %ld (0x%lx)\n",
595 __func__, __LINE__, ko->ko_name, (long)map_data_base,
596 (long)ko->ko_data_address, (long)map_data_size,
597 (long)ko->ko_data_address + map_data_size);
598 }
599 if (map_rodata_base != ko->ko_rodata_address + map_rodata_size) {
600 panic("%s:%d: %s: map_rodata_base 0x%lx != address %lx "
601 "+ map_rodata_size %ld (0x%lx)\n",
602 __func__, __LINE__, ko->ko_name, (long)map_rodata_base,
603 (long)ko->ko_rodata_address, (long)map_rodata_size,
604 (long)ko->ko_rodata_address + map_rodata_size);
605 }
606
607 /*
608 * Perform local relocations only. Relocations relating to global
609 * symbols will be done by kobj_affix().
610 */
611 error = kobj_checksyms(ko, false);
612 if (error)
613 goto out;
614
615 error = kobj_relocate(ko, true);
616 if (error)
617 goto out;
618 out:
619 if (hdr != NULL) {
620 kobj_free(ko, hdr, sizeof(*hdr));
621 }
622 kobj_close(ko);
623 if (error != 0) {
624 kobj_unload(ko);
625 }
626
627 return error;
628 }
629
630 static void
kobj_unload_notify(kobj_t ko,vaddr_t addr,size_t size,const char * note)631 kobj_unload_notify(kobj_t ko, vaddr_t addr, size_t size, const char *note)
632 {
633 if (addr == 0)
634 return;
635
636 int error = kobj_machdep(ko, (void *)addr, size, false);
637 if (error)
638 kobj_error(ko, "machine dependent deinit failed (%s) %d",
639 note, error);
640 }
641
642 #define KOBJ_SEGMENT_NOTIFY(ko, what) \
643 kobj_unload_notify(ko, (ko)->ko_ ## what ## _address, \
644 (ko)->ko_ ## what ## _size, # what);
645
646 #define KOBJ_SEGMENT_FREE(ko, what) \
647 do \
648 if ((ko)->ko_ ## what ## _address != 0) \
649 uvm_km_free(module_map, (ko)->ko_ ## what ## _address, \
650 round_page((ko)->ko_ ## what ## _size), UVM_KMF_WIRED); \
651 while (/*CONSTCOND*/ 0)
652
653 /*
654 * kobj_unload:
655 *
656 * Unload an object previously loaded by kobj_load().
657 */
658 void
kobj_unload(kobj_t ko)659 kobj_unload(kobj_t ko)
660 {
661 kobj_close(ko);
662 kobj_jettison(ko);
663
664
665 /*
666 * Notify MD code that a module has been unloaded.
667 */
668 if (ko->ko_loaded) {
669 KOBJ_SEGMENT_NOTIFY(ko, text);
670 KOBJ_SEGMENT_NOTIFY(ko, data);
671 KOBJ_SEGMENT_NOTIFY(ko, rodata);
672 }
673
674 KOBJ_SEGMENT_FREE(ko, text);
675 KOBJ_SEGMENT_FREE(ko, data);
676 KOBJ_SEGMENT_FREE(ko, rodata);
677
678 if (ko->ko_ksyms == true) {
679 ksyms_modunload(ko->ko_name);
680 }
681 if (ko->ko_symtab != NULL) {
682 kobj_free(ko, ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym));
683 }
684 if (ko->ko_strtab != NULL) {
685 kobj_free(ko, ko->ko_strtab, ko->ko_strtabsz);
686 }
687 if (ko->ko_progtab != NULL) {
688 kobj_free(ko, ko->ko_progtab, ko->ko_nprogtab *
689 sizeof(*ko->ko_progtab));
690 ko->ko_progtab = NULL;
691 }
692 if (ko->ko_shstrtab) {
693 kobj_free(ko, ko->ko_shstrtab, ko->ko_shstrtabsz);
694 ko->ko_shstrtab = NULL;
695 }
696
697 kmem_free(ko, sizeof(*ko));
698 }
699
700 /*
701 * kobj_stat:
702 *
703 * Return size and load address of an object.
704 */
705 int
kobj_stat(kobj_t ko,vaddr_t * address,size_t * size)706 kobj_stat(kobj_t ko, vaddr_t *address, size_t *size)
707 {
708
709 if (address != NULL) {
710 *address = ko->ko_text_address;
711 }
712 if (size != NULL) {
713 *size = ko->ko_text_size;
714 }
715 return 0;
716 }
717
718 /*
719 * kobj_affix:
720 *
721 * Set an object's name and perform global relocs. May only be
722 * called after the module and any requisite modules are loaded.
723 */
724 int
kobj_affix(kobj_t ko,const char * name)725 kobj_affix(kobj_t ko, const char *name)
726 {
727 int error;
728
729 KASSERT(ko->ko_ksyms == false);
730 KASSERT(ko->ko_loaded == false);
731
732 kobj_setname(ko, name);
733
734 /* Cache addresses of undefined symbols. */
735 error = kobj_checksyms(ko, true);
736 if (error)
737 goto out;
738
739 /* Now do global relocations. */
740 error = kobj_relocate(ko, false);
741 if (error)
742 goto out;
743
744 /*
745 * Now that we know the name, register the symbol table.
746 * Do after global relocations because ksyms will pack
747 * the table.
748 */
749 ksyms_modload(ko->ko_name, ko->ko_symtab,
750 ko->ko_symcnt * sizeof(Elf_Sym), ko->ko_strtab, ko->ko_strtabsz);
751 ko->ko_ksyms = true;
752
753 /* Jettison unneeded memory post-link. */
754 kobj_jettison(ko);
755
756 /*
757 * Notify MD code that a module has been loaded.
758 *
759 * Most architectures use this opportunity to flush their caches.
760 */
761 if (ko->ko_text_address != 0) {
762 error = kobj_machdep(ko, (void *)ko->ko_text_address,
763 ko->ko_text_size, true);
764 if (error) {
765 kobj_error(ko, "machine dependent init failed (text)"
766 " %d", error);
767 goto out;
768 }
769 }
770
771 if (ko->ko_data_address != 0) {
772 error = kobj_machdep(ko, (void *)ko->ko_data_address,
773 ko->ko_data_size, true);
774 if (error) {
775 kobj_error(ko, "machine dependent init failed (data)"
776 " %d", error);
777 goto out;
778 }
779 }
780
781 if (ko->ko_rodata_address != 0) {
782 error = kobj_machdep(ko, (void *)ko->ko_rodata_address,
783 ko->ko_rodata_size, true);
784 if (error) {
785 kobj_error(ko, "machine dependent init failed (rodata)"
786 " %d", error);
787 goto out;
788 }
789 }
790
791 ko->ko_loaded = true;
792
793 /* Change the memory protections, when needed. */
794 if (ko->ko_text_address != 0) {
795 uvm_km_protect(module_map, ko->ko_text_address,
796 ko->ko_text_size, VM_PROT_READ|VM_PROT_EXECUTE);
797 }
798 if (ko->ko_rodata_address != 0) {
799 uvm_km_protect(module_map, ko->ko_rodata_address,
800 ko->ko_rodata_size, VM_PROT_READ);
801 }
802
803 /* Success! */
804 error = 0;
805
806 out: if (error) {
807 /* If there was an error, destroy the whole object. */
808 kobj_unload(ko);
809 }
810 return error;
811 }
812
813 /*
814 * kobj_find_section:
815 *
816 * Given a section name, search the loaded object and return
817 * virtual address if present and loaded.
818 */
819 int
kobj_find_section(kobj_t ko,const char * name,void ** addr,size_t * size)820 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size)
821 {
822 int i;
823
824 KASSERT(ko->ko_progtab != NULL);
825
826 for (i = 0; i < ko->ko_nprogtab; i++) {
827 if (strcmp(ko->ko_progtab[i].name, name) == 0) {
828 if (addr != NULL) {
829 *addr = ko->ko_progtab[i].addr;
830 }
831 if (size != NULL) {
832 *size = ko->ko_progtab[i].size;
833 }
834 return 0;
835 }
836 }
837
838 return ENOENT;
839 }
840
841 /*
842 * kobj_jettison:
843 *
844 * Release object data not needed after performing relocations.
845 */
846 static void
kobj_jettison(kobj_t ko)847 kobj_jettison(kobj_t ko)
848 {
849 int i;
850
851 if (ko->ko_reltab != NULL) {
852 for (i = 0; i < ko->ko_nrel; i++) {
853 if (ko->ko_reltab[i].rel) {
854 kobj_free(ko, ko->ko_reltab[i].rel,
855 ko->ko_reltab[i].size);
856 }
857 }
858 kobj_free(ko, ko->ko_reltab, ko->ko_nrel *
859 sizeof(*ko->ko_reltab));
860 ko->ko_reltab = NULL;
861 ko->ko_nrel = 0;
862 }
863 if (ko->ko_relatab != NULL) {
864 for (i = 0; i < ko->ko_nrela; i++) {
865 if (ko->ko_relatab[i].rela) {
866 kobj_free(ko, ko->ko_relatab[i].rela,
867 ko->ko_relatab[i].size);
868 }
869 }
870 kobj_free(ko, ko->ko_relatab, ko->ko_nrela *
871 sizeof(*ko->ko_relatab));
872 ko->ko_relatab = NULL;
873 ko->ko_nrela = 0;
874 }
875 if (ko->ko_shdr != NULL) {
876 kobj_free(ko, ko->ko_shdr, ko->ko_shdrsz);
877 ko->ko_shdr = NULL;
878 }
879 }
880
881 const Elf_Sym *
kobj_symbol(kobj_t ko,uintptr_t symidx)882 kobj_symbol(kobj_t ko, uintptr_t symidx)
883 {
884
885 return ko->ko_symtab + symidx;
886 }
887
888
889 /*
890 * kobj_sym_lookup:
891 *
892 * Symbol lookup function to be used when the symbol index
893 * is known (ie during relocation).
894 */
895 int
kobj_sym_lookup(kobj_t ko,uintptr_t symidx,Elf_Addr * val)896 kobj_sym_lookup(kobj_t ko, uintptr_t symidx, Elf_Addr *val)
897 {
898 const Elf_Sym *sym;
899 const char *symbol;
900
901 sym = ko->ko_symtab + symidx;
902
903 if (symidx == SHN_ABS || symidx == 0) {
904 *val = (uintptr_t)sym->st_value;
905 return 0;
906 } else if (symidx >= ko->ko_symcnt) {
907 /*
908 * Don't even try to lookup the symbol if the index is
909 * bogus.
910 */
911 kobj_error(ko, "symbol index %ju out of range",
912 (uintmax_t)symidx);
913 return EINVAL;
914 }
915
916 /* Quick answer if there is a definition included. */
917 if (sym->st_shndx != SHN_UNDEF) {
918 *val = (uintptr_t)sym->st_value;
919 return 0;
920 }
921
922 /* If we get here, then it is undefined and needs a lookup. */
923 switch (ELF_ST_BIND(sym->st_info)) {
924 case STB_LOCAL:
925 /* Local, but undefined? huh? */
926 kobj_error(ko, "local symbol @%ju undefined",
927 (uintmax_t)symidx);
928 return EINVAL;
929
930 case STB_GLOBAL:
931 /* Relative to Data or Function name */
932 symbol = ko->ko_strtab + sym->st_name;
933
934 /* Force a lookup failure if the symbol name is bogus. */
935 if (*symbol == 0) {
936 kobj_error(ko, "bad symbol @%ju name",
937 (uintmax_t)symidx);
938 return EINVAL;
939 }
940 if (sym->st_value == 0) {
941 kobj_error(ko, "%s @%ju: bad value", symbol,
942 (uintmax_t)symidx);
943 return EINVAL;
944 }
945
946 *val = (uintptr_t)sym->st_value;
947 return 0;
948
949 case STB_WEAK:
950 kobj_error(ko, "weak symbol @%ju not supported",
951 (uintmax_t)symidx);
952 return EINVAL;
953
954 default:
955 kobj_error(ko, "bad binding %#x for symbol @%ju",
956 ELF_ST_BIND(sym->st_info), (uintmax_t)symidx);
957 return EINVAL;
958 }
959 }
960
961 /*
962 * kobj_findbase:
963 *
964 * Return base address of the given section.
965 */
966 static uintptr_t
kobj_findbase(kobj_t ko,int sec)967 kobj_findbase(kobj_t ko, int sec)
968 {
969 int i;
970
971 for (i = 0; i < ko->ko_nprogtab; i++) {
972 if (sec == ko->ko_progtab[i].sec) {
973 return (uintptr_t)ko->ko_progtab[i].addr;
974 }
975 }
976 return 0;
977 }
978
979 /*
980 * kobj_checksyms:
981 *
982 * Scan symbol table for duplicates or resolve references to
983 * external symbols.
984 */
985 static int
kobj_checksyms(kobj_t ko,bool undefined)986 kobj_checksyms(kobj_t ko, bool undefined)
987 {
988 unsigned long rval;
989 Elf_Sym *sym, *ksym, *ms;
990 const char *name;
991 int error;
992
993 error = 0;
994
995 for (ms = (sym = ko->ko_symtab) + ko->ko_symcnt; sym < ms; sym++) {
996 /* Check validity of the symbol. */
997 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL ||
998 sym->st_name == 0)
999 continue;
1000 if (undefined != (sym->st_shndx == SHN_UNDEF)) {
1001 continue;
1002 }
1003
1004 /*
1005 * Look it up. Don't need to lock, as it is known that
1006 * the symbol tables aren't going to change (we hold
1007 * module_lock).
1008 */
1009 name = ko->ko_strtab + sym->st_name;
1010 if (ksyms_getval_unlocked(NULL, name, &ksym, &rval,
1011 KSYMS_EXTERN) != 0) {
1012 if (undefined) {
1013 kobj_error(ko, "symbol `%s' not found",
1014 name);
1015 error = ENOEXEC;
1016 }
1017 continue;
1018 }
1019
1020 /* Save values of undefined globals. */
1021 if (undefined) {
1022 if (ksym->st_shndx == SHN_ABS) {
1023 sym->st_shndx = SHN_ABS;
1024 }
1025 sym->st_value = (Elf_Addr)rval;
1026 continue;
1027 }
1028
1029 /* Check (and complain) about differing values. */
1030 if (sym->st_value == rval) {
1031 continue;
1032 }
1033 if (strcmp(name, "_bss_start") == 0 ||
1034 strcmp(name, "__bss_start") == 0 ||
1035 strcmp(name, "_bss_end__") == 0 ||
1036 strcmp(name, "__bss_end__") == 0 ||
1037 strcmp(name, "_edata") == 0 ||
1038 strcmp(name, "_end") == 0 ||
1039 strcmp(name, "__end") == 0 ||
1040 strcmp(name, "__end__") == 0 ||
1041 strncmp(name, "__start_link_set_", 17) == 0 ||
1042 strncmp(name, "__stop_link_set_", 16) == 0) {
1043 continue;
1044 }
1045 kobj_error(ko, "global symbol `%s' redefined",
1046 name);
1047 error = ENOEXEC;
1048 }
1049
1050 return error;
1051 }
1052
1053 /*
1054 * kobj_relocate:
1055 *
1056 * Resolve relocations for the loaded object.
1057 */
1058 static int
kobj_relocate(kobj_t ko,bool local)1059 kobj_relocate(kobj_t ko, bool local)
1060 {
1061 const Elf_Rel *rellim;
1062 const Elf_Rel *rel;
1063 const Elf_Rela *relalim;
1064 const Elf_Rela *rela;
1065 const Elf_Sym *sym;
1066 uintptr_t base;
1067 int i, error;
1068 uintptr_t symidx;
1069
1070 /*
1071 * Perform relocations without addend if there are any.
1072 */
1073 for (i = 0; i < ko->ko_nrel; i++) {
1074 rel = ko->ko_reltab[i].rel;
1075 if (rel == NULL) {
1076 continue;
1077 }
1078 rellim = rel + ko->ko_reltab[i].nrel;
1079 base = kobj_findbase(ko, ko->ko_reltab[i].sec);
1080 if (base == 0) {
1081 panic("%s:%d: %s: lost base for e_reltab[%d] sec %d",
1082 __func__, __LINE__, ko->ko_name, i,
1083 ko->ko_reltab[i].sec);
1084 }
1085 for (; rel < rellim; rel++) {
1086 symidx = ELF_R_SYM(rel->r_info);
1087 if (symidx >= ko->ko_symcnt) {
1088 continue;
1089 }
1090 sym = ko->ko_symtab + symidx;
1091 /* Skip non-local symbols in the first pass (local == TRUE) */
1092 if (local && (ELF_ST_BIND(sym->st_info) != STB_LOCAL)) {
1093 continue;
1094 }
1095 error = kobj_reloc(ko, base, rel, false, local);
1096 if (error != 0) {
1097 kobj_error(ko, "unresolved rel relocation "
1098 "@%#jx type=%d symidx=%d",
1099 (intmax_t)rel->r_offset,
1100 (int)ELF_R_TYPE(rel->r_info),
1101 (int)ELF_R_SYM(rel->r_info));
1102 return ENOEXEC;
1103 }
1104 }
1105 }
1106
1107 /*
1108 * Perform relocations with addend if there are any.
1109 */
1110 for (i = 0; i < ko->ko_nrela; i++) {
1111 rela = ko->ko_relatab[i].rela;
1112 if (rela == NULL) {
1113 continue;
1114 }
1115 relalim = rela + ko->ko_relatab[i].nrela;
1116 base = kobj_findbase(ko, ko->ko_relatab[i].sec);
1117 if (base == 0) {
1118 panic("%s:%d: %s: lost base for e_relatab[%d] sec %d",
1119 __func__, __LINE__, ko->ko_name, i,
1120 ko->ko_relatab[i].sec);
1121 }
1122 for (; rela < relalim; rela++) {
1123 symidx = ELF_R_SYM(rela->r_info);
1124 if (symidx >= ko->ko_symcnt) {
1125 continue;
1126 }
1127 sym = ko->ko_symtab + symidx;
1128 /* Skip non-local symbols in the first pass (local == TRUE) */
1129 if (local && (ELF_ST_BIND(sym->st_info) != STB_LOCAL)) {
1130 continue;
1131 }
1132 error = kobj_reloc(ko, base, rela, true, local);
1133 if (error != 0) {
1134 kobj_error(ko, "unresolved rela relocation "
1135 "@%#jx type=%d symidx=%d",
1136 (intmax_t)rela->r_offset,
1137 (int)ELF_R_TYPE(rela->r_info),
1138 (int)ELF_R_SYM(rela->r_info));
1139 return ENOEXEC;
1140 }
1141 }
1142 }
1143
1144 return 0;
1145 }
1146
1147 /*
1148 * kobj_out:
1149 *
1150 * Utility function: log an error.
1151 */
1152 static void
kobj_out(const char * fname,int lnum,kobj_t ko,const char * fmt,...)1153 kobj_out(const char *fname, int lnum, kobj_t ko, const char *fmt, ...)
1154 {
1155 va_list ap;
1156
1157 printf("%s, %d: [%s]: linker error: ", fname, lnum, ko->ko_name);
1158 va_start(ap, fmt);
1159 vprintf(fmt, ap);
1160 va_end(ap);
1161 printf("\n");
1162 }
1163
1164 static int
kobj_read_mem(kobj_t ko,void ** basep,size_t size,off_t off,bool allocate)1165 kobj_read_mem(kobj_t ko, void **basep, size_t size, off_t off,
1166 bool allocate)
1167 {
1168 void *base = *basep;
1169 int error = 0;
1170
1171 KASSERT(ko->ko_source != NULL);
1172
1173 if (off < 0) {
1174 kobj_error(ko, "negative offset %lld",
1175 (unsigned long long)off);
1176 error = EINVAL;
1177 base = NULL;
1178 goto out;
1179 } else if (ko->ko_memsize != -1 &&
1180 (size > ko->ko_memsize || off > ko->ko_memsize - size)) {
1181 kobj_error(ko, "preloaded object short");
1182 error = EINVAL;
1183 base = NULL;
1184 goto out;
1185 }
1186
1187 if (allocate)
1188 base = kmem_alloc(size, KM_SLEEP);
1189
1190 /* Copy the section */
1191 memcpy(base, (uint8_t *)ko->ko_source + off, size);
1192
1193 out: if (allocate)
1194 *basep = base;
1195 return error;
1196 }
1197
1198 /*
1199 * kobj_free:
1200 *
1201 * Utility function: free memory if it was allocated from the heap.
1202 */
1203 static void
kobj_free(kobj_t ko,void * base,size_t size)1204 kobj_free(kobj_t ko, void *base, size_t size)
1205 {
1206
1207 kmem_free(base, size);
1208 }
1209
1210 void
kobj_setname(kobj_t ko,const char * name)1211 kobj_setname(kobj_t ko, const char *name)
1212 {
1213 const char *d = name, *dots = "";
1214 size_t len, dlen;
1215
1216 for (char *s = module_base; *d == *s; d++, s++)
1217 continue;
1218
1219 if (d == name)
1220 name = "";
1221 else
1222 name = "%M";
1223 dlen = strlen(d);
1224 len = dlen + strlen(name);
1225 if (len >= sizeof(ko->ko_name)) {
1226 len = (len - sizeof(ko->ko_name)) + 5; /* dots + NUL */
1227 if (dlen >= len) {
1228 d += len;
1229 dots = "/...";
1230 }
1231 }
1232 snprintf(ko->ko_name, sizeof(ko->ko_name), "%s%s%s", name, dots, d);
1233 }
1234
1235 #else /* MODULAR */
1236
1237 int
kobj_load_mem(kobj_t * kop,const char * name,void * base,ssize_t size)1238 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size)
1239 {
1240
1241 return ENOSYS;
1242 }
1243
1244 void
kobj_unload(kobj_t ko)1245 kobj_unload(kobj_t ko)
1246 {
1247
1248 panic("not modular");
1249 }
1250
1251 int
kobj_stat(kobj_t ko,vaddr_t * base,size_t * size)1252 kobj_stat(kobj_t ko, vaddr_t *base, size_t *size)
1253 {
1254
1255 return ENOSYS;
1256 }
1257
1258 int
kobj_affix(kobj_t ko,const char * name)1259 kobj_affix(kobj_t ko, const char *name)
1260 {
1261
1262 panic("not modular");
1263 }
1264
1265 int
kobj_find_section(kobj_t ko,const char * name,void ** addr,size_t * size)1266 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size)
1267 {
1268
1269 panic("not modular");
1270 }
1271
1272 void
kobj_setname(kobj_t ko,const char * name)1273 kobj_setname(kobj_t ko, const char *name)
1274 {
1275
1276 panic("not modular");
1277 }
1278
1279 #endif /* MODULAR */
1280