1 /* $NetBSD: subr_asan.c,v 1.28 2023/04/09 09:18:09 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KASAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.28 2023/04/09 09:18:09 riastradh Exp $");
33
34 #include <sys/param.h>
35 #include <sys/device.h>
36 #include <sys/kernel.h>
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/systm.h>
40 #include <sys/types.h>
41 #include <sys/asan.h>
42
43 #include <uvm/uvm_extern.h>
44
45 #ifdef DDB
46 #include <machine/db_machdep.h>
47 #include <ddb/db_extern.h>
48 #endif
49
50 #ifdef KASAN_PANIC
51 #define REPORT panic
52 #else
53 #define REPORT printf
54 #endif
55
56 /* ASAN constants. Part of the compiler ABI. */
57 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
58 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1)
59 #define KASAN_ALLOCA_SCALE_SIZE 32
60
61 /* The MD code. */
62 #include <machine/asan.h>
63
64 /* ASAN ABI version. */
65 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
66 #define ASAN_ABI_VERSION 8
67 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
68 #define ASAN_ABI_VERSION 8
69 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
70 #define ASAN_ABI_VERSION 6
71 #else
72 #error "Unsupported compiler version"
73 #endif
74
75 #define __RET_ADDR (unsigned long)__builtin_return_address(0)
76
77 /* Global variable descriptor. Part of the compiler ABI. */
78 struct __asan_global_source_location {
79 const char *filename;
80 int line_no;
81 int column_no;
82 };
83 struct __asan_global {
84 const void *beg; /* address of the global variable */
85 size_t size; /* size of the global variable */
86 size_t size_with_redzone; /* size with the redzone */
87 const void *name; /* name of the variable */
88 const void *module_name; /* name of the module where the var is declared */
89 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
90 struct __asan_global_source_location *location;
91 #if ASAN_ABI_VERSION >= 7
92 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
93 #endif
94 };
95
96 static bool kasan_enabled __read_mostly = false;
97
98 /* -------------------------------------------------------------------------- */
99
100 void
kasan_shadow_map(void * addr,size_t size)101 kasan_shadow_map(void *addr, size_t size)
102 {
103 size_t sz, npages, i;
104 vaddr_t sva, eva;
105
106 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
107
108 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
109
110 sva = (vaddr_t)kasan_md_addr_to_shad(addr);
111 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
112
113 sva = rounddown(sva, PAGE_SIZE);
114 eva = roundup(eva, PAGE_SIZE);
115
116 npages = (eva - sva) / PAGE_SIZE;
117
118 KASSERT(sva >= KASAN_MD_SHADOW_START);
119 KASSERT(eva < KASAN_MD_SHADOW_END);
120
121 for (i = 0; i < npages; i++) {
122 kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
123 }
124 }
125
126 static void
kasan_ctors(void)127 kasan_ctors(void)
128 {
129 extern Elf_Addr __CTOR_LIST__, __CTOR_END__;
130 size_t nentries, i;
131 Elf_Addr *ptr;
132
133 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
134 sizeof(uintptr_t);
135
136 ptr = &__CTOR_LIST__;
137 for (i = 0; i < nentries; i++) {
138 void (*func)(void);
139
140 func = (void *)(*ptr);
141 (*func)();
142
143 ptr++;
144 }
145 }
146
147 void
kasan_early_init(void * stack)148 kasan_early_init(void *stack)
149 {
150 kasan_md_early_init(stack);
151 }
152
153 void
kasan_init(void)154 kasan_init(void)
155 {
156 /* MD initialization. */
157 kasan_md_init();
158
159 /* Now officially enabled. */
160 kasan_enabled = true;
161
162 /* Call the ASAN constructors. */
163 kasan_ctors();
164 }
165
166 static inline const char *
kasan_code_name(uint8_t code)167 kasan_code_name(uint8_t code)
168 {
169 switch (code) {
170 case KASAN_GENERIC_REDZONE:
171 return "GenericRedZone";
172 case KASAN_MALLOC_REDZONE:
173 return "MallocRedZone";
174 case KASAN_KMEM_REDZONE:
175 return "KmemRedZone";
176 case KASAN_POOL_REDZONE:
177 return "PoolRedZone";
178 case KASAN_POOL_FREED:
179 return "PoolUseAfterFree";
180 case 1 ... 7:
181 return "RedZonePartial";
182 case KASAN_STACK_LEFT:
183 return "StackLeft";
184 case KASAN_STACK_MID:
185 return "StackMiddle";
186 case KASAN_STACK_RIGHT:
187 return "StackRight";
188 case KASAN_USE_AFTER_RET:
189 return "UseAfterRet";
190 case KASAN_USE_AFTER_SCOPE:
191 return "UseAfterScope";
192 default:
193 return "Unknown";
194 }
195 }
196
197 static void
kasan_report(unsigned long addr,size_t size,bool write,unsigned long pc,uint8_t code)198 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
199 uint8_t code)
200 {
201 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s,"
202 " %s]\n",
203 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
204 (write ? "write" : "read"), kasan_code_name(code));
205 kasan_md_unwind();
206 }
207
208 static __always_inline void
kasan_shadow_1byte_markvalid(unsigned long addr)209 kasan_shadow_1byte_markvalid(unsigned long addr)
210 {
211 int8_t *byte = kasan_md_addr_to_shad((void *)addr);
212 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
213
214 *byte = last;
215 }
216
217 static __always_inline void
kasan_shadow_Nbyte_markvalid(const void * addr,size_t size)218 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
219 {
220 size_t i;
221
222 for (i = 0; i < size; i++) {
223 kasan_shadow_1byte_markvalid((unsigned long)addr+i);
224 }
225 }
226
227 static __always_inline void
kasan_shadow_Nbyte_fill(const void * addr,size_t size,uint8_t code)228 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
229 {
230 void *shad;
231
232 if (__predict_false(size == 0))
233 return;
234 if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
235 return;
236
237 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
238 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
239
240 shad = (void *)kasan_md_addr_to_shad(addr);
241 size = size >> KASAN_SHADOW_SCALE_SHIFT;
242
243 __builtin_memset(shad, code, size);
244 }
245
246 void
kasan_add_redzone(size_t * size)247 kasan_add_redzone(size_t *size)
248 {
249 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
250 *size += KASAN_SHADOW_SCALE_SIZE;
251 }
252
253 void
kasan_softint(struct lwp * l)254 kasan_softint(struct lwp *l)
255 {
256 const void *stk = (const void *)uvm_lwp_getuarea(l);
257
258 kasan_shadow_Nbyte_fill(stk, USPACE, 0);
259 }
260
261 /*
262 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
263 * and the rest as invalid. There are generally two use cases:
264 *
265 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
266 * the redzone at the end of the buffer as invalid.
267 *
268 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
269 */
270 void
kasan_mark(const void * addr,size_t size,size_t sz_with_redz,uint8_t code)271 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code)
272 {
273 size_t i, n, redz;
274 int8_t *shad;
275
276 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
277 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE);
278 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0);
279 shad = kasan_md_addr_to_shad(addr);
280
281 /* Chunks of 8 bytes, valid. */
282 n = size / KASAN_SHADOW_SCALE_SIZE;
283 for (i = 0; i < n; i++) {
284 *shad++ = 0;
285 }
286
287 /* Possibly one chunk, mid. */
288 if ((size & KASAN_SHADOW_MASK) != 0) {
289 *shad++ = (size & KASAN_SHADOW_MASK);
290 }
291
292 /* Chunks of 8 bytes, invalid. */
293 n = redz / KASAN_SHADOW_SCALE_SIZE;
294 for (i = 0; i < n; i++) {
295 *shad++ = code;
296 }
297 }
298
299 /* -------------------------------------------------------------------------- */
300
301 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
302 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
303 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
304
305 static __always_inline bool
kasan_shadow_1byte_isvalid(unsigned long addr,uint8_t * code)306 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
307 {
308 int8_t *byte = kasan_md_addr_to_shad((void *)addr);
309 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
310
311 if (__predict_true(*byte == 0 || last <= *byte)) {
312 return true;
313 }
314 *code = *byte;
315 return false;
316 }
317
318 static __always_inline bool
kasan_shadow_2byte_isvalid(unsigned long addr,uint8_t * code)319 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
320 {
321 int8_t *byte, last;
322
323 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
324 return (kasan_shadow_1byte_isvalid(addr, code) &&
325 kasan_shadow_1byte_isvalid(addr+1, code));
326 }
327
328 byte = kasan_md_addr_to_shad((void *)addr);
329 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
330
331 if (__predict_true(*byte == 0 || last <= *byte)) {
332 return true;
333 }
334 *code = *byte;
335 return false;
336 }
337
338 static __always_inline bool
kasan_shadow_4byte_isvalid(unsigned long addr,uint8_t * code)339 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
340 {
341 int8_t *byte, last;
342
343 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
344 return (kasan_shadow_2byte_isvalid(addr, code) &&
345 kasan_shadow_2byte_isvalid(addr+2, code));
346 }
347
348 byte = kasan_md_addr_to_shad((void *)addr);
349 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
350
351 if (__predict_true(*byte == 0 || last <= *byte)) {
352 return true;
353 }
354 *code = *byte;
355 return false;
356 }
357
358 static __always_inline bool
kasan_shadow_8byte_isvalid(unsigned long addr,uint8_t * code)359 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
360 {
361 int8_t *byte, last;
362
363 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
364 return (kasan_shadow_4byte_isvalid(addr, code) &&
365 kasan_shadow_4byte_isvalid(addr+4, code));
366 }
367
368 byte = kasan_md_addr_to_shad((void *)addr);
369 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
370
371 if (__predict_true(*byte == 0 || last <= *byte)) {
372 return true;
373 }
374 *code = *byte;
375 return false;
376 }
377
378 static __always_inline bool
kasan_shadow_Nbyte_isvalid(unsigned long addr,size_t size,uint8_t * code)379 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
380 {
381 size_t i;
382
383 for (i = 0; i < size; i++) {
384 if (!kasan_shadow_1byte_isvalid(addr+i, code))
385 return false;
386 }
387
388 return true;
389 }
390
391 static __always_inline void
kasan_shadow_check(unsigned long addr,size_t size,bool write,unsigned long retaddr)392 kasan_shadow_check(unsigned long addr, size_t size, bool write,
393 unsigned long retaddr)
394 {
395 uint8_t code;
396 bool valid;
397
398 if (__predict_false(!kasan_enabled))
399 return;
400 #ifdef DDB
401 if (__predict_false(db_recover != NULL))
402 return;
403 #endif
404 if (__predict_false(size == 0))
405 return;
406 if (__predict_false(kasan_md_unsupported(addr)))
407 return;
408
409 if (__builtin_constant_p(size)) {
410 switch (size) {
411 case 1:
412 valid = kasan_shadow_1byte_isvalid(addr, &code);
413 break;
414 case 2:
415 valid = kasan_shadow_2byte_isvalid(addr, &code);
416 break;
417 case 4:
418 valid = kasan_shadow_4byte_isvalid(addr, &code);
419 break;
420 case 8:
421 valid = kasan_shadow_8byte_isvalid(addr, &code);
422 break;
423 default:
424 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
425 break;
426 }
427 } else {
428 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
429 }
430
431 if (__predict_false(!valid)) {
432 kasan_report(addr, size, write, retaddr, code);
433 }
434 }
435
436 /* -------------------------------------------------------------------------- */
437
438 void *
kasan_memcpy(void * dst,const void * src,size_t len)439 kasan_memcpy(void *dst, const void *src, size_t len)
440 {
441 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
442 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
443 return __builtin_memcpy(dst, src, len);
444 }
445
446 int
kasan_memcmp(const void * b1,const void * b2,size_t len)447 kasan_memcmp(const void *b1, const void *b2, size_t len)
448 {
449 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
450 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
451 return __builtin_memcmp(b1, b2, len);
452 }
453
454 void *
kasan_memset(void * b,int c,size_t len)455 kasan_memset(void *b, int c, size_t len)
456 {
457 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
458 return __builtin_memset(b, c, len);
459 }
460
461 void *
kasan_memmove(void * dst,const void * src,size_t len)462 kasan_memmove(void *dst, const void *src, size_t len)
463 {
464 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
465 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
466 return __builtin_memmove(dst, src, len);
467 }
468
469 char *
kasan_strcpy(char * dst,const char * src)470 kasan_strcpy(char *dst, const char *src)
471 {
472 char *save = dst;
473
474 while (1) {
475 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
476 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
477 *dst = *src;
478 if (*src == '\0')
479 break;
480 src++, dst++;
481 }
482
483 return save;
484 }
485
486 int
kasan_strcmp(const char * s1,const char * s2)487 kasan_strcmp(const char *s1, const char *s2)
488 {
489 while (1) {
490 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
491 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
492 if (*s1 != *s2)
493 break;
494 if (*s1 == '\0')
495 return 0;
496 s1++, s2++;
497 }
498
499 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
500 }
501
502 size_t
kasan_strlen(const char * str)503 kasan_strlen(const char *str)
504 {
505 const char *s;
506
507 s = str;
508 while (1) {
509 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
510 if (*s == '\0')
511 break;
512 s++;
513 }
514
515 return (s - str);
516 }
517
518 char *
kasan_strcat(char * dst,const char * src)519 kasan_strcat(char *dst, const char *src)
520 {
521 size_t ldst, lsrc;
522
523 ldst = __builtin_strlen(dst);
524 lsrc = __builtin_strlen(src);
525 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true,
526 __RET_ADDR);
527 kasan_shadow_check((unsigned long)src, lsrc + 1, false,
528 __RET_ADDR);
529
530 return __builtin_strcat(dst, src);
531 }
532
533 char *
kasan_strchr(const char * s,int c)534 kasan_strchr(const char *s, int c)
535 {
536 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
537 __RET_ADDR);
538 return __builtin_strchr(s, c);
539 }
540
541 char *
kasan_strrchr(const char * s,int c)542 kasan_strrchr(const char *s, int c)
543 {
544 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
545 __RET_ADDR);
546 return __builtin_strrchr(s, c);
547 }
548
549 #undef kcopy
550 #undef copyinstr
551 #undef copyoutstr
552 #undef copyin
553
554 int kasan_kcopy(const void *, void *, size_t);
555 int kasan_copyinstr(const void *, void *, size_t, size_t *);
556 int kasan_copyoutstr(const void *, void *, size_t, size_t *);
557 int kasan_copyin(const void *, void *, size_t);
558 int kcopy(const void *, void *, size_t);
559 int copyinstr(const void *, void *, size_t, size_t *);
560 int copyoutstr(const void *, void *, size_t, size_t *);
561 int copyin(const void *, void *, size_t);
562
563 int
kasan_kcopy(const void * src,void * dst,size_t len)564 kasan_kcopy(const void *src, void *dst, size_t len)
565 {
566 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
567 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
568 return kcopy(src, dst, len);
569 }
570
571 int
kasan_copyin(const void * uaddr,void * kaddr,size_t len)572 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
573 {
574 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
575 return copyin(uaddr, kaddr, len);
576 }
577
578 int
kasan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)579 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
580 {
581 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
582 return copyinstr(uaddr, kaddr, len, done);
583 }
584
585 int
kasan_copyoutstr(const void * kaddr,void * uaddr,size_t len,size_t * done)586 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
587 {
588 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
589 return copyoutstr(kaddr, uaddr, len, done);
590 }
591
592 /* -------------------------------------------------------------------------- */
593
594 #undef _ucas_32
595 #undef _ucas_32_mp
596 #undef _ucas_64
597 #undef _ucas_64_mp
598 #undef _ufetch_8
599 #undef _ufetch_16
600 #undef _ufetch_32
601 #undef _ufetch_64
602
603 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
604 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
605 int
kasan__ucas_32(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)606 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
607 uint32_t *ret)
608 {
609 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
610 __RET_ADDR);
611 return _ucas_32(uaddr, old, new, ret);
612 }
613
614 #ifdef __HAVE_UCAS_MP
615 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
616 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
617 int
kasan__ucas_32_mp(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)618 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
619 uint32_t *ret)
620 {
621 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
622 __RET_ADDR);
623 return _ucas_32_mp(uaddr, old, new, ret);
624 }
625 #endif
626
627 #ifdef _LP64
628 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
629 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
630 int
kasan__ucas_64(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)631 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
632 uint64_t *ret)
633 {
634 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
635 __RET_ADDR);
636 return _ucas_64(uaddr, old, new, ret);
637 }
638
639 #ifdef __HAVE_UCAS_MP
640 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
641 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
642 int
kasan__ucas_64_mp(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)643 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
644 uint64_t *ret)
645 {
646 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
647 __RET_ADDR);
648 return _ucas_64_mp(uaddr, old, new, ret);
649 }
650 #endif
651 #endif
652
653 int _ufetch_8(const uint8_t *, uint8_t *);
654 int kasan__ufetch_8(const uint8_t *, uint8_t *);
655 int
kasan__ufetch_8(const uint8_t * uaddr,uint8_t * valp)656 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
657 {
658 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
659 __RET_ADDR);
660 return _ufetch_8(uaddr, valp);
661 }
662
663 int _ufetch_16(const uint16_t *, uint16_t *);
664 int kasan__ufetch_16(const uint16_t *, uint16_t *);
665 int
kasan__ufetch_16(const uint16_t * uaddr,uint16_t * valp)666 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
667 {
668 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
669 __RET_ADDR);
670 return _ufetch_16(uaddr, valp);
671 }
672
673 int _ufetch_32(const uint32_t *, uint32_t *);
674 int kasan__ufetch_32(const uint32_t *, uint32_t *);
675 int
kasan__ufetch_32(const uint32_t * uaddr,uint32_t * valp)676 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
677 {
678 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
679 __RET_ADDR);
680 return _ufetch_32(uaddr, valp);
681 }
682
683 #ifdef _LP64
684 int _ufetch_64(const uint64_t *, uint64_t *);
685 int kasan__ufetch_64(const uint64_t *, uint64_t *);
686 int
kasan__ufetch_64(const uint64_t * uaddr,uint64_t * valp)687 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
688 {
689 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
690 __RET_ADDR);
691 return _ufetch_64(uaddr, valp);
692 }
693 #endif
694
695 /* -------------------------------------------------------------------------- */
696
697 #undef atomic_add_32
698 #undef atomic_add_int
699 #undef atomic_add_long
700 #undef atomic_add_ptr
701 #undef atomic_add_64
702 #undef atomic_add_32_nv
703 #undef atomic_add_int_nv
704 #undef atomic_add_long_nv
705 #undef atomic_add_ptr_nv
706 #undef atomic_add_64_nv
707 #undef atomic_and_32
708 #undef atomic_and_uint
709 #undef atomic_and_ulong
710 #undef atomic_and_64
711 #undef atomic_and_32_nv
712 #undef atomic_and_uint_nv
713 #undef atomic_and_ulong_nv
714 #undef atomic_and_64_nv
715 #undef atomic_or_32
716 #undef atomic_or_uint
717 #undef atomic_or_ulong
718 #undef atomic_or_64
719 #undef atomic_or_32_nv
720 #undef atomic_or_uint_nv
721 #undef atomic_or_ulong_nv
722 #undef atomic_or_64_nv
723 #undef atomic_cas_32
724 #undef atomic_cas_uint
725 #undef atomic_cas_ulong
726 #undef atomic_cas_ptr
727 #undef atomic_cas_64
728 #undef atomic_cas_32_ni
729 #undef atomic_cas_uint_ni
730 #undef atomic_cas_ulong_ni
731 #undef atomic_cas_ptr_ni
732 #undef atomic_cas_64_ni
733 #undef atomic_swap_32
734 #undef atomic_swap_uint
735 #undef atomic_swap_ulong
736 #undef atomic_swap_ptr
737 #undef atomic_swap_64
738 #undef atomic_dec_32
739 #undef atomic_dec_uint
740 #undef atomic_dec_ulong
741 #undef atomic_dec_ptr
742 #undef atomic_dec_64
743 #undef atomic_dec_32_nv
744 #undef atomic_dec_uint_nv
745 #undef atomic_dec_ulong_nv
746 #undef atomic_dec_ptr_nv
747 #undef atomic_dec_64_nv
748 #undef atomic_inc_32
749 #undef atomic_inc_uint
750 #undef atomic_inc_ulong
751 #undef atomic_inc_ptr
752 #undef atomic_inc_64
753 #undef atomic_inc_32_nv
754 #undef atomic_inc_uint_nv
755 #undef atomic_inc_ulong_nv
756 #undef atomic_inc_ptr_nv
757 #undef atomic_inc_64_nv
758
759 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
760 void atomic_add_##name(volatile targ1 *, targ2); \
761 void kasan_atomic_add_##name(volatile targ1 *, targ2); \
762 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
763 { \
764 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
765 __RET_ADDR); \
766 atomic_add_##name(ptr, val); \
767 } \
768 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
769 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
770 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
771 { \
772 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
773 __RET_ADDR); \
774 return atomic_add_##name##_nv(ptr, val); \
775 }
776
777 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
778 void atomic_and_##name(volatile targ1 *, targ2); \
779 void kasan_atomic_and_##name(volatile targ1 *, targ2); \
780 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
781 { \
782 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
783 __RET_ADDR); \
784 atomic_and_##name(ptr, val); \
785 } \
786 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
787 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
788 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
789 { \
790 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
791 __RET_ADDR); \
792 return atomic_and_##name##_nv(ptr, val); \
793 }
794
795 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
796 void atomic_or_##name(volatile targ1 *, targ2); \
797 void kasan_atomic_or_##name(volatile targ1 *, targ2); \
798 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
799 { \
800 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
801 __RET_ADDR); \
802 atomic_or_##name(ptr, val); \
803 } \
804 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
805 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
806 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
807 { \
808 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
809 __RET_ADDR); \
810 return atomic_or_##name##_nv(ptr, val); \
811 }
812
813 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
814 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
815 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
816 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
817 { \
818 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
819 __RET_ADDR); \
820 return atomic_cas_##name(ptr, exp, new); \
821 } \
822 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
823 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
824 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
825 { \
826 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
827 __RET_ADDR); \
828 return atomic_cas_##name##_ni(ptr, exp, new); \
829 }
830
831 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
832 tret atomic_swap_##name(volatile targ1 *, targ2); \
833 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \
834 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
835 { \
836 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
837 __RET_ADDR); \
838 return atomic_swap_##name(ptr, val); \
839 }
840
841 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
842 void atomic_dec_##name(volatile targ1 *); \
843 void kasan_atomic_dec_##name(volatile targ1 *); \
844 void kasan_atomic_dec_##name(volatile targ1 *ptr) \
845 { \
846 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
847 __RET_ADDR); \
848 atomic_dec_##name(ptr); \
849 } \
850 tret atomic_dec_##name##_nv(volatile targ1 *); \
851 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \
852 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
853 { \
854 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
855 __RET_ADDR); \
856 return atomic_dec_##name##_nv(ptr); \
857 }
858
859 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \
860 void atomic_inc_##name(volatile targ1 *); \
861 void kasan_atomic_inc_##name(volatile targ1 *); \
862 void kasan_atomic_inc_##name(volatile targ1 *ptr) \
863 { \
864 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
865 __RET_ADDR); \
866 atomic_inc_##name(ptr); \
867 } \
868 tret atomic_inc_##name##_nv(volatile targ1 *); \
869 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \
870 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
871 { \
872 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
873 __RET_ADDR); \
874 return atomic_inc_##name##_nv(ptr); \
875 }
876
877 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
878 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
879 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
880 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
881 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
882
883 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
884 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
885 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
886 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
887
888 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
889 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
890 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
891 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
892
893 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
894 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
895 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
896 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
897 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
898
899 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
900 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
901 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
902 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
903 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
904
905 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
906 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
907 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
908 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
909 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void);
910
911 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
912 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
913 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
914 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
915 ASAN_ATOMIC_FUNC_INC(ptr, void *, void);
916
917 /* -------------------------------------------------------------------------- */
918
919 #ifdef __HAVE_KASAN_INSTR_BUS
920
921 #include <sys/bus.h>
922
923 #undef bus_space_read_multi_1
924 #undef bus_space_read_multi_2
925 #undef bus_space_read_multi_4
926 #undef bus_space_read_multi_8
927 #undef bus_space_read_multi_stream_1
928 #undef bus_space_read_multi_stream_2
929 #undef bus_space_read_multi_stream_4
930 #undef bus_space_read_multi_stream_8
931 #undef bus_space_read_region_1
932 #undef bus_space_read_region_2
933 #undef bus_space_read_region_4
934 #undef bus_space_read_region_8
935 #undef bus_space_read_region_stream_1
936 #undef bus_space_read_region_stream_2
937 #undef bus_space_read_region_stream_4
938 #undef bus_space_read_region_stream_8
939 #undef bus_space_write_multi_1
940 #undef bus_space_write_multi_2
941 #undef bus_space_write_multi_4
942 #undef bus_space_write_multi_8
943 #undef bus_space_write_multi_stream_1
944 #undef bus_space_write_multi_stream_2
945 #undef bus_space_write_multi_stream_4
946 #undef bus_space_write_multi_stream_8
947 #undef bus_space_write_region_1
948 #undef bus_space_write_region_2
949 #undef bus_space_write_region_4
950 #undef bus_space_write_region_8
951 #undef bus_space_write_region_stream_1
952 #undef bus_space_write_region_stream_2
953 #undef bus_space_write_region_stream_4
954 #undef bus_space_write_region_stream_8
955
956 #define ASAN_BUS_READ_FUNC(bytes, bits) \
957 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
958 bus_size_t, uint##bits##_t *, bus_size_t); \
959 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \
960 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
961 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
962 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
963 bus_size_t count) \
964 { \
965 kasan_shadow_check((uintptr_t)buf, \
966 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
967 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
968 } \
969 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
970 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
971 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
972 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
973 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
974 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
975 bus_size_t count) \
976 { \
977 kasan_shadow_check((uintptr_t)buf, \
978 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
979 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
980 } \
981 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
982 bus_size_t, uint##bits##_t *, bus_size_t); \
983 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \
984 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
985 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
986 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
987 bus_size_t count) \
988 { \
989 kasan_shadow_check((uintptr_t)buf, \
990 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
991 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
992 } \
993 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
994 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
995 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
996 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
997 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
998 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
999 bus_size_t count) \
1000 { \
1001 kasan_shadow_check((uintptr_t)buf, \
1002 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
1003 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1004 }
1005
1006 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \
1007 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1008 bus_size_t, const uint##bits##_t *, bus_size_t); \
1009 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1010 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1011 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1012 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1013 bus_size_t count) \
1014 { \
1015 kasan_shadow_check((uintptr_t)buf, \
1016 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1017 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1018 } \
1019 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1020 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1021 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1022 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1023 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1024 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1025 bus_size_t count) \
1026 { \
1027 kasan_shadow_check((uintptr_t)buf, \
1028 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1029 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1030 } \
1031 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1032 bus_size_t, const uint##bits##_t *, bus_size_t); \
1033 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \
1034 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1035 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1036 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1037 bus_size_t count) \
1038 { \
1039 kasan_shadow_check((uintptr_t)buf, \
1040 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1041 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1042 } \
1043 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1044 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1045 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1046 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1047 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1048 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1049 bus_size_t count) \
1050 { \
1051 kasan_shadow_check((uintptr_t)buf, \
1052 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1053 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1054 }
1055
1056 ASAN_BUS_READ_FUNC(1, 8)
1057 ASAN_BUS_READ_FUNC(2, 16)
1058 ASAN_BUS_READ_FUNC(4, 32)
1059 ASAN_BUS_READ_FUNC(8, 64)
1060
1061 ASAN_BUS_WRITE_FUNC(1, 8)
1062 ASAN_BUS_WRITE_FUNC(2, 16)
1063 ASAN_BUS_WRITE_FUNC(4, 32)
1064 ASAN_BUS_WRITE_FUNC(8, 64)
1065
1066 #endif /* __HAVE_KASAN_INSTR_BUS */
1067
1068 /* -------------------------------------------------------------------------- */
1069
1070 #include <sys/mbuf.h>
1071
1072 static void
kasan_dma_sync_linear(uint8_t * buf,bus_addr_t offset,bus_size_t len,bool write,uintptr_t pc)1073 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1074 bool write, uintptr_t pc)
1075 {
1076 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc);
1077 }
1078
1079 static void
kasan_dma_sync_mbuf(struct mbuf * m,bus_addr_t offset,bus_size_t len,bool write,uintptr_t pc)1080 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1081 bool write, uintptr_t pc)
1082 {
1083 bus_addr_t minlen;
1084
1085 for (; m != NULL && len != 0; m = m->m_next) {
1086 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc);
1087
1088 if (offset >= m->m_len) {
1089 offset -= m->m_len;
1090 continue;
1091 }
1092
1093 minlen = MIN(len, m->m_len - offset);
1094 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset),
1095 minlen, write, pc);
1096
1097 offset = 0;
1098 len -= minlen;
1099 }
1100 }
1101
1102 static void
kasan_dma_sync_uio(struct uio * uio,bus_addr_t offset,bus_size_t len,bool write,uintptr_t pc)1103 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1104 bool write, uintptr_t pc)
1105 {
1106 bus_size_t minlen, resid;
1107 struct iovec *iov;
1108 int i;
1109
1110 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc);
1111
1112 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
1113 return;
1114
1115 resid = uio->uio_resid;
1116 iov = uio->uio_iov;
1117
1118 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1119 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]),
1120 false, pc);
1121 minlen = MIN(resid, iov[i].iov_len);
1122 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen,
1123 write, pc);
1124 resid -= minlen;
1125 }
1126 }
1127
1128 void
kasan_dma_sync(bus_dmamap_t map,bus_addr_t offset,bus_size_t len,int ops)1129 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1130 {
1131 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0;
1132
1133 switch (map->dm_buftype) {
1134 case KASAN_DMA_LINEAR:
1135 kasan_dma_sync_linear(map->dm_buf, offset, len, write,
1136 __RET_ADDR);
1137 break;
1138 case KASAN_DMA_MBUF:
1139 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write,
1140 __RET_ADDR);
1141 break;
1142 case KASAN_DMA_UIO:
1143 kasan_dma_sync_uio(map->dm_buf, offset, len, write,
1144 __RET_ADDR);
1145 break;
1146 case KASAN_DMA_RAW:
1147 break;
1148 default:
1149 panic("%s: impossible", __func__);
1150 }
1151 }
1152
1153 void
kasan_dma_load(bus_dmamap_t map,void * buf,bus_size_t buflen,int type)1154 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1155 {
1156 map->dm_buf = buf;
1157 map->dm_buflen = buflen;
1158 map->dm_buftype = type;
1159 }
1160
1161 /* -------------------------------------------------------------------------- */
1162
1163 void __asan_register_globals(struct __asan_global *, size_t);
1164 void __asan_unregister_globals(struct __asan_global *, size_t);
1165
1166 void
__asan_register_globals(struct __asan_global * globals,size_t n)1167 __asan_register_globals(struct __asan_global *globals, size_t n)
1168 {
1169 size_t i;
1170
1171 for (i = 0; i < n; i++) {
1172 kasan_mark(globals[i].beg, globals[i].size,
1173 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1174 }
1175 }
1176
1177 void
__asan_unregister_globals(struct __asan_global * globals,size_t n)1178 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1179 {
1180 /* never called */
1181 }
1182
1183 #define ASAN_LOAD_STORE(size) \
1184 void __asan_load##size(unsigned long); \
1185 void __asan_load##size(unsigned long addr) \
1186 { \
1187 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1188 } \
1189 void __asan_load##size##_noabort(unsigned long); \
1190 void __asan_load##size##_noabort(unsigned long addr) \
1191 { \
1192 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1193 } \
1194 void __asan_store##size(unsigned long); \
1195 void __asan_store##size(unsigned long addr) \
1196 { \
1197 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1198 } \
1199 void __asan_store##size##_noabort(unsigned long); \
1200 void __asan_store##size##_noabort(unsigned long addr) \
1201 { \
1202 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1203 }
1204
1205 ASAN_LOAD_STORE(1);
1206 ASAN_LOAD_STORE(2);
1207 ASAN_LOAD_STORE(4);
1208 ASAN_LOAD_STORE(8);
1209 ASAN_LOAD_STORE(16);
1210
1211 void __asan_loadN(unsigned long, size_t);
1212 void __asan_loadN_noabort(unsigned long, size_t);
1213 void __asan_storeN(unsigned long, size_t);
1214 void __asan_storeN_noabort(unsigned long, size_t);
1215 void __asan_handle_no_return(void);
1216
1217 void
__asan_loadN(unsigned long addr,size_t size)1218 __asan_loadN(unsigned long addr, size_t size)
1219 {
1220 kasan_shadow_check(addr, size, false, __RET_ADDR);
1221 }
1222
1223 void
__asan_loadN_noabort(unsigned long addr,size_t size)1224 __asan_loadN_noabort(unsigned long addr, size_t size)
1225 {
1226 kasan_shadow_check(addr, size, false, __RET_ADDR);
1227 }
1228
1229 void
__asan_storeN(unsigned long addr,size_t size)1230 __asan_storeN(unsigned long addr, size_t size)
1231 {
1232 kasan_shadow_check(addr, size, true, __RET_ADDR);
1233 }
1234
1235 void
__asan_storeN_noabort(unsigned long addr,size_t size)1236 __asan_storeN_noabort(unsigned long addr, size_t size)
1237 {
1238 kasan_shadow_check(addr, size, true, __RET_ADDR);
1239 }
1240
1241 void
__asan_handle_no_return(void)1242 __asan_handle_no_return(void)
1243 {
1244 /* nothing */
1245 }
1246
1247 #define ASAN_SET_SHADOW(byte) \
1248 void __asan_set_shadow_##byte(void *, size_t); \
1249 void __asan_set_shadow_##byte(void *addr, size_t size) \
1250 { \
1251 __builtin_memset((void *)addr, 0x##byte, size); \
1252 }
1253
1254 ASAN_SET_SHADOW(00);
1255 ASAN_SET_SHADOW(f1);
1256 ASAN_SET_SHADOW(f2);
1257 ASAN_SET_SHADOW(f3);
1258 ASAN_SET_SHADOW(f5);
1259 ASAN_SET_SHADOW(f8);
1260
1261 void __asan_poison_stack_memory(const void *, size_t);
1262 void __asan_unpoison_stack_memory(const void *, size_t);
1263
1264 void
__asan_poison_stack_memory(const void * addr,size_t size)1265 __asan_poison_stack_memory(const void *addr, size_t size)
1266 {
1267 size = roundup(size, KASAN_SHADOW_SCALE_SIZE);
1268 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1269 }
1270
1271 void
__asan_unpoison_stack_memory(const void * addr,size_t size)1272 __asan_unpoison_stack_memory(const void *addr, size_t size)
1273 {
1274 kasan_shadow_Nbyte_markvalid(addr, size);
1275 }
1276
1277 void __asan_alloca_poison(const void *, size_t);
1278 void __asan_allocas_unpoison(const void *, const void *);
1279
__asan_alloca_poison(const void * addr,size_t size)1280 void __asan_alloca_poison(const void *addr, size_t size)
1281 {
1282 const void *l, *r;
1283
1284 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0);
1285
1286 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1287 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1288
1289 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1290 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1291 KASAN_STACK_MID);
1292 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1293 }
1294
__asan_allocas_unpoison(const void * stkbegin,const void * stkend)1295 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1296 {
1297 size_t size;
1298
1299 if (__predict_false(!stkbegin))
1300 return;
1301 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1302 return;
1303 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1304
1305 kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1306 }
1307