1 /* $NetBSD: subr_msan.c,v 1.19 2023/04/11 10:19:56 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KMSAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.19 2023/04/11 10:19:56 riastradh Exp $");
33
34 #include <sys/param.h>
35 #include <sys/device.h>
36 #include <sys/kernel.h>
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/systm.h>
40 #include <sys/types.h>
41 #include <sys/kprintf.h>
42 #include <sys/kmem.h>
43 #include <sys/mbuf.h>
44 #include <sys/buf.h>
45 #include <sys/cpu.h>
46 #include <sys/msan.h>
47
48 #include <ddb/db_active.h>
49
50 static void kmsan_printf(const char *, ...);
51
52 void kmsan_init_arg(size_t);
53 void kmsan_init_ret(size_t);
54
55 #ifdef KMSAN_PANIC
56 #define REPORT panic
57 #else
58 #define REPORT kmsan_printf
59 #endif
60
61 /* -------------------------------------------------------------------------- */
62
63 /*
64 * Part of the compiler ABI.
65 */
66
67 typedef uint32_t msan_orig_t;
68
69 typedef struct {
70 uint8_t *shad;
71 msan_orig_t *orig;
72 } msan_meta_t;
73
74 #define MSAN_PARAM_SIZE 800
75 #define MSAN_RETVAL_SIZE 800
76 typedef struct {
77 uint8_t param[MSAN_PARAM_SIZE];
78 uint8_t retval[MSAN_RETVAL_SIZE];
79 uint8_t _va_arg[MSAN_PARAM_SIZE];
80 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
81 uint64_t va_arg_overflow_size;
82 msan_orig_t param_origin[MSAN_PARAM_SIZE];
83 msan_orig_t retval_origin;
84 msan_orig_t origin;
85 } msan_tls_t;
86
87 /* -------------------------------------------------------------------------- */
88
89 /* The MD code. */
90 #include <machine/msan.h>
91
92 /* -------------------------------------------------------------------------- */
93
94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
95 #define MSAN_NCONTEXT 16
96
97 typedef struct {
98 size_t ctx;
99 msan_tls_t tls[MSAN_NCONTEXT];
100 } msan_lwp_t;
101
102 static msan_tls_t dummy_tls;
103
104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
106 static msan_lwp_t msan_lwp0;
107 static bool kmsan_enabled __read_mostly;
108
109 /* -------------------------------------------------------------------------- */
110
111 static bool kmsan_reporting = false;
112
113 static inline void
kmsan_printf(const char * fmt,...)114 kmsan_printf(const char *fmt, ...)
115 {
116 va_list ap;
117
118 va_start(ap, fmt);
119 kprintf(fmt, TOCONS, NULL, NULL, ap);
120 va_end(ap);
121 }
122
123 static inline const char *
kmsan_orig_name(int type)124 kmsan_orig_name(int type)
125 {
126 switch (type) {
127 case KMSAN_TYPE_STACK:
128 return "Stack";
129 case KMSAN_TYPE_KMEM:
130 return "Kmem";
131 case KMSAN_TYPE_MALLOC:
132 return "Malloc";
133 case KMSAN_TYPE_POOL:
134 return "Pool";
135 case KMSAN_TYPE_UVM:
136 return "Uvm";
137 default:
138 return "Unknown";
139 }
140 }
141
142 /*
143 * The format of the string is: "----var@function". Parse it to display a nice
144 * warning.
145 */
146 static void
kmsan_report_hook(const void * addr,size_t size,size_t off,const char * hook)147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
148 {
149 unsigned long symstart;
150 const char *mod, *sym;
151 msan_orig_t *orig;
152 const char *typename;
153 char *var, *fn;
154 uintptr_t ptr;
155 char buf[128];
156 int type;
157 int s;
158
159 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
160 return;
161
162 kmsan_reporting = true;
163 __insn_barrier();
164
165 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
166 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
167
168 if (*orig == 0) {
169 REPORT("MSan: Uninitialized Memory In %s At Offset "
170 "%zu\n", hook, off);
171 goto out;
172 }
173
174 kmsan_md_orig_decode(*orig, &type, &ptr);
175 typename = kmsan_orig_name(type);
176
177 if (kmsan_md_is_pc(ptr)) {
178 s = pserialize_read_enter();
179 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC) ||
180 ksyms_getval(mod, sym, &symstart, KSYMS_PROC)) {
181 REPORT("MSan: Uninitialized %s Memory In %s "
182 "At Offset %zu/%zu, IP %p\n", typename, hook, off,
183 size, (void *)ptr);
184 } else {
185 char soff[16] = "";
186
187 if ((vaddr_t)ptr < symstart) {
188 snprintf(soff, sizeof(soff), "-0x%"PRIxVADDR,
189 symstart - (vaddr_t)ptr);
190 } else if ((vaddr_t)ptr > symstart) {
191 snprintf(soff, sizeof(soff), "+0x%"PRIxVADDR,
192 (vaddr_t)ptr - symstart);
193 }
194 REPORT("MSan: Uninitialized %s Memory In %s "
195 "At Offset %zu/%zu, From %s%s%lx\n",
196 typename, hook,
197 off, size, sym,
198 ((unsigned long)ptr < symstart ? "-" :
199 (unsigned long)ptr > symstart ? "+" :
200 ""),
201 (unsigned long)ptr - symstart);
202 }
203 pserialize_read_exit(s);
204 } else {
205 var = (char *)ptr + 4;
206 strlcpy(buf, var, sizeof(buf));
207 var = buf;
208 fn = __builtin_strchr(buf, '@');
209 *fn++ = '\0';
210 REPORT("MSan: Uninitialized %s Memory In %s At Offset "
211 "%zu, Variable '%s' From %s()\n", typename, hook, off,
212 var, fn);
213 }
214
215 out:
216 kmsan_md_unwind();
217 __insn_barrier();
218 kmsan_reporting = false;
219 }
220
221 static void
kmsan_report_inline(msan_orig_t orig,unsigned long pc)222 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
223 {
224 const char *mod, *sym;
225 const char *typename;
226 char *var, *fn;
227 uintptr_t ptr;
228 char buf[128];
229 int type;
230 int s;
231
232 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting))
233 return;
234
235 kmsan_reporting = true;
236 __insn_barrier();
237
238 if (orig == 0) {
239 REPORT("MSan: Uninitialized Variable In %p\n",
240 (void *)pc);
241 goto out;
242 }
243
244 kmsan_md_orig_decode(orig, &type, &ptr);
245 typename = kmsan_orig_name(type);
246
247 if (kmsan_md_is_pc(ptr)) {
248 s = pserialize_read_enter();
249 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) {
250 REPORT("MSan: Uninitialized %s Memory, "
251 "Origin %x\n", typename, orig);
252 } else {
253 REPORT("MSan: Uninitialized %s Memory "
254 "From %s()\n", typename, sym);
255 }
256 pserialize_read_exit(s);
257 } else {
258 var = (char *)ptr + 4;
259 strlcpy(buf, var, sizeof(buf));
260 var = buf;
261 fn = __builtin_strchr(buf, '@');
262 *fn++ = '\0';
263 REPORT("MSan: Uninitialized Variable '%s' From %s()\n",
264 var, fn);
265 }
266
267 out:
268 kmsan_md_unwind();
269 __insn_barrier();
270 kmsan_reporting = false;
271 }
272
273 /* -------------------------------------------------------------------------- */
274
275 static inline msan_meta_t
kmsan_meta_get(void * addr,size_t size)276 kmsan_meta_get(void *addr, size_t size)
277 {
278 msan_meta_t ret;
279
280 if (__predict_false(!kmsan_enabled)) {
281 ret.shad = msan_dummy_shad;
282 ret.orig = (msan_orig_t *)msan_dummy_orig;
283 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) {
284 ret.shad = msan_dummy_shad;
285 ret.orig = (msan_orig_t *)msan_dummy_orig;
286 } else {
287 ret.shad = (void *)kmsan_md_addr_to_shad(addr);
288 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3);
290 }
291
292 return ret;
293 }
294
295 static inline void
kmsan_origin_fill(void * addr,msan_orig_t o,size_t size)296 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size)
297 {
298 msan_orig_t *orig;
299 size_t i;
300
301 if (__predict_false(!kmsan_enabled))
302 return;
303 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
304 return;
305
306 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr);
307 size += ((uintptr_t)orig & 0x3);
308 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3);
309
310 for (i = 0; i < size; i += 4) {
311 orig[i / 4] = o;
312 }
313 }
314
315 static inline void
kmsan_shadow_fill(void * addr,uint8_t c,size_t size)316 kmsan_shadow_fill(void *addr, uint8_t c, size_t size)
317 {
318 uint8_t *shad;
319
320 if (__predict_false(!kmsan_enabled))
321 return;
322 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
323 return;
324
325 shad = kmsan_md_addr_to_shad(addr);
326 __builtin_memset(shad, c, size);
327 }
328
329 static inline void
kmsan_meta_copy(void * dst,const void * src,size_t size)330 kmsan_meta_copy(void *dst, const void *src, size_t size)
331 {
332 uint8_t *orig_src, *orig_dst;
333 uint8_t *shad_src, *shad_dst;
334 msan_orig_t *_src, *_dst;
335 size_t i;
336
337 if (__predict_false(!kmsan_enabled))
338 return;
339 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst)))
340 return;
341 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) {
342 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size);
343 return;
344 }
345
346 shad_src = kmsan_md_addr_to_shad(src);
347 shad_dst = kmsan_md_addr_to_shad(dst);
348 __builtin_memmove(shad_dst, shad_src, size);
349
350 orig_src = kmsan_md_addr_to_orig(src);
351 orig_dst = kmsan_md_addr_to_orig(dst);
352 for (i = 0; i < size; i++) {
353 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3);
354 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3);
355 *_dst = *_src;
356 orig_src++;
357 orig_dst++;
358 }
359 }
360
361 static inline void
kmsan_shadow_check(const void * addr,size_t size,const char * hook)362 kmsan_shadow_check(const void *addr, size_t size, const char *hook)
363 {
364 uint8_t *shad;
365 size_t i;
366
367 if (__predict_false(!kmsan_enabled))
368 return;
369 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr)))
370 return;
371
372 shad = kmsan_md_addr_to_shad(addr);
373 for (i = 0; i < size; i++) {
374 if (__predict_true(shad[i] == 0))
375 continue;
376 kmsan_report_hook((const char *)addr + i, size, i, hook);
377 break;
378 }
379 }
380
381 void
kmsan_init_arg(size_t n)382 kmsan_init_arg(size_t n)
383 {
384 msan_lwp_t *lwp;
385 uint8_t *arg;
386
387 if (__predict_false(!kmsan_enabled))
388 return;
389 lwp = curlwp->l_kmsan;
390 arg = lwp->tls[lwp->ctx].param;
391 __builtin_memset(arg, 0, n);
392 }
393
394 void
kmsan_init_ret(size_t n)395 kmsan_init_ret(size_t n)
396 {
397 msan_lwp_t *lwp;
398 uint8_t *arg;
399
400 if (__predict_false(!kmsan_enabled))
401 return;
402 lwp = curlwp->l_kmsan;
403 arg = lwp->tls[lwp->ctx].retval;
404 __builtin_memset(arg, 0, n);
405 }
406
407 static void
kmsan_check_arg(size_t size,const char * hook)408 kmsan_check_arg(size_t size, const char *hook)
409 {
410 msan_lwp_t *lwp;
411 uint8_t *arg;
412 size_t i;
413
414 if (__predict_false(!kmsan_enabled))
415 return;
416 lwp = curlwp->l_kmsan;
417 arg = lwp->tls[lwp->ctx].param;
418
419 for (i = 0; i < size; i++) {
420 if (__predict_true(arg[i] == 0))
421 continue;
422 kmsan_report_hook((const char *)arg + i, size, i, hook);
423 break;
424 }
425 }
426
427 void
kmsan_lwp_alloc(struct lwp * l)428 kmsan_lwp_alloc(struct lwp *l)
429 {
430 msan_lwp_t *lwp;
431
432 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t));
433 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP);
434 lwp->ctx = 1;
435
436 l->l_kmsan = lwp;
437 }
438
439 void
kmsan_lwp_free(struct lwp * l)440 kmsan_lwp_free(struct lwp *l)
441 {
442 kmsan_init_arg(sizeof(void *) + sizeof(size_t));
443 kmem_free(l->l_kmsan, sizeof(msan_lwp_t));
444 }
445
446 void kmsan_intr_enter(void);
447 void kmsan_intr_leave(void);
448 void kmsan_softint(struct lwp *);
449
450 void
kmsan_intr_enter(void)451 kmsan_intr_enter(void)
452 {
453 msan_lwp_t *lwp;
454
455 if (__predict_false(!kmsan_enabled))
456 return;
457 lwp = curlwp->l_kmsan;
458
459 lwp->ctx++;
460 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) {
461 kmsan_enabled = false;
462 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
463 }
464
465 kmsan_init_arg(sizeof(void *));
466 }
467
468 void
kmsan_intr_leave(void)469 kmsan_intr_leave(void)
470 {
471 msan_lwp_t *lwp;
472
473 if (__predict_false(!kmsan_enabled))
474 return;
475 lwp = curlwp->l_kmsan;
476
477 if (__predict_false(lwp->ctx == 0)) {
478 kmsan_enabled = false;
479 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx);
480 }
481 lwp->ctx--;
482 }
483
484 void
kmsan_softint(struct lwp * l)485 kmsan_softint(struct lwp *l)
486 {
487 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int));
488 }
489
490 /* -------------------------------------------------------------------------- */
491
492 void
kmsan_shadow_map(void * addr,size_t size)493 kmsan_shadow_map(void *addr, size_t size)
494 {
495 size_t npages, i;
496 vaddr_t va;
497
498 KASSERT((vaddr_t)addr % PAGE_SIZE == 0);
499 KASSERT(size % PAGE_SIZE == 0);
500
501 npages = size / PAGE_SIZE;
502
503 va = (vaddr_t)kmsan_md_addr_to_shad(addr);
504 for (i = 0; i < npages; i++) {
505 kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
506 }
507
508 va = (vaddr_t)kmsan_md_addr_to_orig(addr);
509 for (i = 0; i < npages; i++) {
510 kmsan_md_shadow_map_page(va + i * PAGE_SIZE);
511 }
512 }
513
514 void
kmsan_orig(void * addr,size_t size,int type,uintptr_t pc)515 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc)
516 {
517 msan_orig_t orig;
518
519 orig = kmsan_md_orig_encode(type, pc);
520 kmsan_origin_fill(addr, orig, size);
521 }
522
523 void
kmsan_mark(void * addr,size_t size,uint8_t c)524 kmsan_mark(void *addr, size_t size, uint8_t c)
525 {
526 kmsan_shadow_fill(addr, c, size);
527 }
528
529 void
kmsan_check_mbuf(void * buf)530 kmsan_check_mbuf(void *buf)
531 {
532 struct mbuf *m = buf;
533
534 do {
535 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain");
536 } while ((m = m->m_next) != NULL);
537 }
538
539 void
kmsan_check_buf(void * buf)540 kmsan_check_buf(void *buf)
541 {
542 buf_t *bp = buf;
543
544 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()");
545 }
546
547 void
kmsan_init(void * stack)548 kmsan_init(void *stack)
549 {
550 /* MD initialization. */
551 kmsan_md_init();
552
553 /* Map the stack. */
554 kmsan_shadow_map(stack, USPACE);
555
556 /* Initialize the TLS for curlwp. */
557 msan_lwp0.ctx = 1;
558 curlwp->l_kmsan = &msan_lwp0;
559
560 /* Now officially enabled. */
561 kmsan_enabled = true;
562 }
563
564 /* -------------------------------------------------------------------------- */
565
566 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
567 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
568
__msan_metadata_ptr_for_load_n(void * addr,size_t size)569 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size)
570 {
571 return kmsan_meta_get(addr, size);
572 }
573
__msan_metadata_ptr_for_store_n(void * addr,size_t size)574 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size)
575 {
576 return kmsan_meta_get(addr, size);
577 }
578
579 #define MSAN_META_FUNC(size) \
580 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
581 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
582 { \
583 return kmsan_meta_get(addr, size); \
584 } \
585 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
586 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
587 { \
588 return kmsan_meta_get(addr, size); \
589 }
590
591 MSAN_META_FUNC(1)
592 MSAN_META_FUNC(2)
593 MSAN_META_FUNC(4)
594 MSAN_META_FUNC(8)
595
596 void __msan_instrument_asm_store(void *, size_t);
597 msan_orig_t __msan_chain_origin(msan_orig_t);
598 void __msan_poison_alloca(void *, uint64_t, char *);
599 void __msan_unpoison_alloca(void *, uint64_t);
600 void __msan_warning(msan_orig_t);
601 msan_tls_t *__msan_get_context_state(void);
602
__msan_instrument_asm_store(void * addr,size_t size)603 void __msan_instrument_asm_store(void *addr, size_t size)
604 {
605 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
606 }
607
__msan_chain_origin(msan_orig_t origin)608 msan_orig_t __msan_chain_origin(msan_orig_t origin)
609 {
610 return origin;
611 }
612
__msan_poison_alloca(void * addr,uint64_t size,char * descr)613 void __msan_poison_alloca(void *addr, uint64_t size, char *descr)
614 {
615 msan_orig_t orig;
616
617 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
618 kmsan_origin_fill(addr, orig, size);
619 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size);
620 }
621
__msan_unpoison_alloca(void * addr,uint64_t size)622 void __msan_unpoison_alloca(void *addr, uint64_t size)
623 {
624 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size);
625 }
626
__msan_warning(msan_orig_t origin)627 void __msan_warning(msan_orig_t origin)
628 {
629 if (__predict_false(!kmsan_enabled))
630 return;
631 kmsan_report_inline(origin, __RET_ADDR);
632 }
633
__msan_get_context_state(void)634 msan_tls_t *__msan_get_context_state(void)
635 {
636 msan_lwp_t *lwp;
637
638 if (__predict_false(!kmsan_enabled))
639 return &dummy_tls;
640 lwp = curlwp->l_kmsan;
641
642 return &lwp->tls[lwp->ctx];
643 }
644
645 /* -------------------------------------------------------------------------- */
646
647 /*
648 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
649 * initialized areas properly.
650 */
651
kmsan_memcpy(void * dst,const void * src,size_t len)652 void *kmsan_memcpy(void *dst, const void *src, size_t len)
653 {
654 /* No kmsan_check_arg, because inlined. */
655 kmsan_init_ret(sizeof(void *));
656 if (__predict_true(len != 0)) {
657 kmsan_meta_copy(dst, src, len);
658 }
659 return __builtin_memcpy(dst, src, len);
660 }
661
662 int
kmsan_memcmp(const void * b1,const void * b2,size_t len)663 kmsan_memcmp(const void *b1, const void *b2, size_t len)
664 {
665 const uint8_t *_b1 = b1, *_b2 = b2;
666 size_t i;
667
668 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
669 "memcmp():args");
670 kmsan_init_ret(sizeof(int));
671
672 for (i = 0; i < len; i++) {
673 if (*_b1 != *_b2) {
674 kmsan_shadow_check(b1, i + 1, "memcmp():arg1");
675 kmsan_shadow_check(b2, i + 1, "memcmp():arg2");
676 return *_b1 - *_b2;
677 }
678 _b1++, _b2++;
679 }
680
681 return 0;
682 }
683
kmsan_memset(void * dst,int c,size_t len)684 void *kmsan_memset(void *dst, int c, size_t len)
685 {
686 /* No kmsan_check_arg, because inlined. */
687 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len);
688 kmsan_init_ret(sizeof(void *));
689 return __builtin_memset(dst, c, len);
690 }
691
kmsan_memmove(void * dst,const void * src,size_t len)692 void *kmsan_memmove(void *dst, const void *src, size_t len)
693 {
694 /* No kmsan_check_arg, because inlined. */
695 if (__predict_true(len != 0)) {
696 kmsan_meta_copy(dst, src, len);
697 }
698 kmsan_init_ret(sizeof(void *));
699 return __builtin_memmove(dst, src, len);
700 }
701
__strong_alias(__msan_memcpy,kmsan_memcpy)702 __strong_alias(__msan_memcpy, kmsan_memcpy)
703 __strong_alias(__msan_memset, kmsan_memset)
704 __strong_alias(__msan_memmove, kmsan_memmove)
705
706 char *
707 kmsan_strcpy(char *dst, const char *src)
708 {
709 const char *_src = src;
710 char *_dst = dst;
711 size_t len = 0;
712
713 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
714
715 while (1) {
716 len++;
717 *dst = *src;
718 if (*src == '\0')
719 break;
720 src++, dst++;
721 }
722
723 kmsan_shadow_check(_src, len, "strcpy():arg2");
724 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len);
725 kmsan_init_ret(sizeof(char *));
726 return _dst;
727 }
728
729 int
kmsan_strcmp(const char * s1,const char * s2)730 kmsan_strcmp(const char *s1, const char *s2)
731 {
732 const char *_s1 = s1, *_s2 = s2;
733 size_t len = 0;
734
735 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
736 kmsan_init_ret(sizeof(int));
737
738 while (1) {
739 len++;
740 if (*s1 != *s2)
741 break;
742 if (*s1 == '\0') {
743 kmsan_shadow_check(_s1, len, "strcmp():arg1");
744 kmsan_shadow_check(_s2, len, "strcmp():arg2");
745 return 0;
746 }
747 s1++, s2++;
748 }
749
750 kmsan_shadow_check(_s1, len, "strcmp():arg1");
751 kmsan_shadow_check(_s2, len, "strcmp():arg2");
752
753 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
754 }
755
756 size_t
kmsan_strlen(const char * str)757 kmsan_strlen(const char *str)
758 {
759 const char *s;
760
761 kmsan_check_arg(sizeof(str), "strlen():args");
762
763 s = str;
764 while (1) {
765 if (*s == '\0')
766 break;
767 s++;
768 }
769
770 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1");
771 kmsan_init_ret(sizeof(size_t));
772 return (s - str);
773 }
774
775 char *
kmsan_strcat(char * dst,const char * src)776 kmsan_strcat(char *dst, const char *src)
777 {
778 size_t ldst, lsrc;
779 char *ret;
780
781 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args");
782
783 ldst = __builtin_strlen(dst);
784 lsrc = __builtin_strlen(src);
785 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1");
786 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2");
787 ret = __builtin_strcat(dst, src);
788 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1);
789
790 kmsan_init_ret(sizeof(char *));
791 return ret;
792 }
793
794 char *
kmsan_strchr(const char * s,int c)795 kmsan_strchr(const char *s, int c)
796 {
797 char *ret;
798
799 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args");
800 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1");
801 ret = __builtin_strchr(s, c);
802
803 kmsan_init_ret(sizeof(char *));
804 return ret;
805 }
806
807 char *
kmsan_strrchr(const char * s,int c)808 kmsan_strrchr(const char *s, int c)
809 {
810 char *ret;
811
812 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args");
813 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1");
814 ret = __builtin_strrchr(s, c);
815
816 kmsan_init_ret(sizeof(char *));
817 return ret;
818 }
819
820 #undef kcopy
821 #undef copyin
822 #undef copyout
823 #undef copyinstr
824 #undef copyoutstr
825
826 int kmsan_kcopy(const void *, void *, size_t);
827 int kmsan_copyin(const void *, void *, size_t);
828 int kmsan_copyout(const void *, void *, size_t);
829 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
830 int kmsan_copyoutstr(const void *, void *, size_t, size_t *);
831
832 int kcopy(const void *, void *, size_t);
833 int copyin(const void *, void *, size_t);
834 int copyout(const void *, void *, size_t);
835 int copyinstr(const void *, void *, size_t, size_t *);
836 int copyoutstr(const void *, void *, size_t, size_t *);
837
838 int
kmsan_kcopy(const void * src,void * dst,size_t len)839 kmsan_kcopy(const void *src, void *dst, size_t len)
840 {
841 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len),
842 "kcopy():args");
843 if (__predict_true(len != 0)) {
844 kmsan_meta_copy(dst, src, len);
845 }
846 kmsan_init_ret(sizeof(int));
847 return kcopy(src, dst, len);
848 }
849
850 int
kmsan_copyin(const void * uaddr,void * kaddr,size_t len)851 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
852 {
853 int ret;
854
855 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
856 "copyin():args");
857 ret = copyin(uaddr, kaddr, len);
858 if (ret == 0)
859 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len);
860 kmsan_init_ret(sizeof(int));
861
862 return ret;
863 }
864
865 int
kmsan_copyout(const void * kaddr,void * uaddr,size_t len)866 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
867 {
868 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
869 "copyout():args");
870 kmsan_shadow_check(kaddr, len, "copyout():arg1");
871 kmsan_init_ret(sizeof(int));
872 return copyout(kaddr, uaddr, len);
873 }
874
875 int
kmsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)876 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
877 {
878 size_t _done;
879 int ret;
880
881 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
882 sizeof(len) + sizeof(done), "copyinstr():args");
883 ret = copyinstr(uaddr, kaddr, len, &_done);
884 if (ret == 0 || ret == ENAMETOOLONG)
885 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done);
886 if (done != NULL) {
887 *done = _done;
888 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
889 }
890 kmsan_init_ret(sizeof(int));
891
892 return ret;
893 }
894
895 int
kmsan_copyoutstr(const void * kaddr,void * uaddr,size_t len,size_t * done)896 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
897 {
898 size_t _done;
899 int ret;
900
901 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) +
902 sizeof(len) + sizeof(done), "copyoutstr():args");
903 ret = copyoutstr(kaddr, uaddr, len, &_done);
904 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1");
905 if (done != NULL) {
906 *done = _done;
907 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t));
908 }
909 kmsan_init_ret(sizeof(int));
910
911 return ret;
912 }
913
914 /* -------------------------------------------------------------------------- */
915
916 #undef _ucas_32
917 #undef _ucas_32_mp
918 #undef _ucas_64
919 #undef _ucas_64_mp
920 #undef _ufetch_8
921 #undef _ufetch_16
922 #undef _ufetch_32
923 #undef _ufetch_64
924 #undef _ustore_8
925 #undef _ustore_16
926 #undef _ustore_32
927 #undef _ustore_64
928
929 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
930 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
931 int
kmsan__ucas_32(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)932 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
933 uint32_t *ret)
934 {
935 int _ret;
936 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
937 sizeof(new) + sizeof(ret), "ucas_32():args");
938 _ret = _ucas_32(uaddr, old, new, ret);
939 if (_ret == 0)
940 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
941 kmsan_init_ret(sizeof(int));
942 return _ret;
943 }
944
945 #ifdef __HAVE_UCAS_MP
946 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
947 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
948 int
kmsan__ucas_32_mp(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)949 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
950 uint32_t *ret)
951 {
952 int _ret;
953 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
954 sizeof(new) + sizeof(ret), "ucas_32_mp():args");
955 _ret = _ucas_32_mp(uaddr, old, new, ret);
956 if (_ret == 0)
957 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
958 kmsan_init_ret(sizeof(int));
959 return _ret;
960 }
961 #endif
962
963 #ifdef _LP64
964 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
965 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
966 int
kmsan__ucas_64(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)967 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
968 uint64_t *ret)
969 {
970 int _ret;
971 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
972 sizeof(new) + sizeof(ret), "ucas_64():args");
973 _ret = _ucas_64(uaddr, old, new, ret);
974 if (_ret == 0)
975 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
976 kmsan_init_ret(sizeof(int));
977 return _ret;
978 }
979
980 #ifdef __HAVE_UCAS_MP
981 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
982 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
983 int
kmsan__ucas_64_mp(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)984 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
985 uint64_t *ret)
986 {
987 int _ret;
988 kmsan_check_arg(sizeof(uaddr) + sizeof(old) +
989 sizeof(new) + sizeof(ret), "ucas_64_mp():args");
990 _ret = _ucas_64_mp(uaddr, old, new, ret);
991 if (_ret == 0)
992 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret));
993 kmsan_init_ret(sizeof(int));
994 return _ret;
995 }
996 #endif
997 #endif
998
999 int _ufetch_8(const uint8_t *, uint8_t *);
1000 int kmsan__ufetch_8(const uint8_t *, uint8_t *);
1001 int
kmsan__ufetch_8(const uint8_t * uaddr,uint8_t * valp)1002 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
1003 {
1004 int _ret;
1005 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args");
1006 _ret = _ufetch_8(uaddr, valp);
1007 if (_ret == 0)
1008 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1009 kmsan_init_ret(sizeof(int));
1010 return _ret;
1011 }
1012
1013 int _ufetch_16(const uint16_t *, uint16_t *);
1014 int kmsan__ufetch_16(const uint16_t *, uint16_t *);
1015 int
kmsan__ufetch_16(const uint16_t * uaddr,uint16_t * valp)1016 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
1017 {
1018 int _ret;
1019 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args");
1020 _ret = _ufetch_16(uaddr, valp);
1021 if (_ret == 0)
1022 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1023 kmsan_init_ret(sizeof(int));
1024 return _ret;
1025 }
1026
1027 int _ufetch_32(const uint32_t *, uint32_t *);
1028 int kmsan__ufetch_32(const uint32_t *, uint32_t *);
1029 int
kmsan__ufetch_32(const uint32_t * uaddr,uint32_t * valp)1030 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
1031 {
1032 int _ret;
1033 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args");
1034 _ret = _ufetch_32(uaddr, valp);
1035 if (_ret == 0)
1036 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1037 kmsan_init_ret(sizeof(int));
1038 return _ret;
1039 }
1040
1041 #ifdef _LP64
1042 int _ufetch_64(const uint64_t *, uint64_t *);
1043 int kmsan__ufetch_64(const uint64_t *, uint64_t *);
1044 int
kmsan__ufetch_64(const uint64_t * uaddr,uint64_t * valp)1045 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
1046 {
1047 int _ret;
1048 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args");
1049 _ret = _ufetch_64(uaddr, valp);
1050 if (_ret == 0)
1051 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp));
1052 kmsan_init_ret(sizeof(int));
1053 return _ret;
1054 }
1055 #endif
1056
1057 int _ustore_8(uint8_t *, uint8_t);
1058 int kmsan__ustore_8(uint8_t *, uint8_t);
1059 int
kmsan__ustore_8(uint8_t * uaddr,uint8_t val)1060 kmsan__ustore_8(uint8_t *uaddr, uint8_t val)
1061 {
1062 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args");
1063 kmsan_init_ret(sizeof(int));
1064 return _ustore_8(uaddr, val);
1065 }
1066
1067 int _ustore_16(uint16_t *, uint16_t);
1068 int kmsan__ustore_16(uint16_t *, uint16_t);
1069 int
kmsan__ustore_16(uint16_t * uaddr,uint16_t val)1070 kmsan__ustore_16(uint16_t *uaddr, uint16_t val)
1071 {
1072 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args");
1073 kmsan_init_ret(sizeof(int));
1074 return _ustore_16(uaddr, val);
1075 }
1076
1077 int _ustore_32(uint32_t *, uint32_t);
1078 int kmsan__ustore_32(uint32_t *, uint32_t);
1079 int
kmsan__ustore_32(uint32_t * uaddr,uint32_t val)1080 kmsan__ustore_32(uint32_t *uaddr, uint32_t val)
1081 {
1082 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args");
1083 kmsan_init_ret(sizeof(int));
1084 return _ustore_32(uaddr, val);
1085 }
1086
1087 #ifdef _LP64
1088 int _ustore_64(uint64_t *, uint64_t);
1089 int kmsan__ustore_64(uint64_t *, uint64_t);
1090 int
kmsan__ustore_64(uint64_t * uaddr,uint64_t val)1091 kmsan__ustore_64(uint64_t *uaddr, uint64_t val)
1092 {
1093 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args");
1094 kmsan_init_ret(sizeof(int));
1095 return _ustore_64(uaddr, val);
1096 }
1097 #endif
1098
1099 /* -------------------------------------------------------------------------- */
1100
1101 #undef atomic_add_32
1102 #undef atomic_add_int
1103 #undef atomic_add_long
1104 #undef atomic_add_ptr
1105 #undef atomic_add_64
1106 #undef atomic_add_32_nv
1107 #undef atomic_add_int_nv
1108 #undef atomic_add_long_nv
1109 #undef atomic_add_ptr_nv
1110 #undef atomic_add_64_nv
1111 #undef atomic_and_32
1112 #undef atomic_and_uint
1113 #undef atomic_and_ulong
1114 #undef atomic_and_64
1115 #undef atomic_and_32_nv
1116 #undef atomic_and_uint_nv
1117 #undef atomic_and_ulong_nv
1118 #undef atomic_and_64_nv
1119 #undef atomic_or_32
1120 #undef atomic_or_uint
1121 #undef atomic_or_ulong
1122 #undef atomic_or_64
1123 #undef atomic_or_32_nv
1124 #undef atomic_or_uint_nv
1125 #undef atomic_or_ulong_nv
1126 #undef atomic_or_64_nv
1127 #undef atomic_cas_32
1128 #undef atomic_cas_uint
1129 #undef atomic_cas_ulong
1130 #undef atomic_cas_ptr
1131 #undef atomic_cas_64
1132 #undef atomic_cas_32_ni
1133 #undef atomic_cas_uint_ni
1134 #undef atomic_cas_ulong_ni
1135 #undef atomic_cas_ptr_ni
1136 #undef atomic_cas_64_ni
1137 #undef atomic_swap_32
1138 #undef atomic_swap_uint
1139 #undef atomic_swap_ulong
1140 #undef atomic_swap_ptr
1141 #undef atomic_swap_64
1142 #undef atomic_dec_32
1143 #undef atomic_dec_uint
1144 #undef atomic_dec_ulong
1145 #undef atomic_dec_ptr
1146 #undef atomic_dec_64
1147 #undef atomic_dec_32_nv
1148 #undef atomic_dec_uint_nv
1149 #undef atomic_dec_ulong_nv
1150 #undef atomic_dec_ptr_nv
1151 #undef atomic_dec_64_nv
1152 #undef atomic_inc_32
1153 #undef atomic_inc_uint
1154 #undef atomic_inc_ulong
1155 #undef atomic_inc_ptr
1156 #undef atomic_inc_64
1157 #undef atomic_inc_32_nv
1158 #undef atomic_inc_uint_nv
1159 #undef atomic_inc_ulong_nv
1160 #undef atomic_inc_ptr_nv
1161 #undef atomic_inc_64_nv
1162
1163 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
1164 void atomic_add_##name(volatile targ1 *, targ2); \
1165 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \
1166 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
1167 { \
1168 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1169 "atomic_add_" #name "():args"); \
1170 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1171 "atomic_add_" #name "():arg1"); \
1172 atomic_add_##name(ptr, val); \
1173 } \
1174 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
1175 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
1176 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
1177 { \
1178 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1179 "atomic_add_" #name "_nv():args"); \
1180 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1181 "atomic_add_" #name "_nv():arg1"); \
1182 kmsan_init_ret(sizeof(tret)); \
1183 return atomic_add_##name##_nv(ptr, val); \
1184 }
1185
1186 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
1187 void atomic_and_##name(volatile targ1 *, targ2); \
1188 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \
1189 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
1190 { \
1191 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1192 "atomic_and_" #name "():args"); \
1193 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1194 "atomic_and_" #name "():arg1"); \
1195 atomic_and_##name(ptr, val); \
1196 } \
1197 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
1198 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
1199 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
1200 { \
1201 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1202 "atomic_and_" #name "_nv():args"); \
1203 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1204 "atomic_and_" #name "_nv():arg1"); \
1205 kmsan_init_ret(sizeof(tret)); \
1206 return atomic_and_##name##_nv(ptr, val); \
1207 }
1208
1209 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
1210 void atomic_or_##name(volatile targ1 *, targ2); \
1211 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \
1212 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
1213 { \
1214 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1215 "atomic_or_" #name "():args"); \
1216 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1217 "atomic_or_" #name "():arg1"); \
1218 atomic_or_##name(ptr, val); \
1219 } \
1220 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
1221 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
1222 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
1223 { \
1224 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1225 "atomic_or_" #name "_nv():args"); \
1226 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1227 "atomic_or_" #name "_nv():arg1"); \
1228 kmsan_init_ret(sizeof(tret)); \
1229 return atomic_or_##name##_nv(ptr, val); \
1230 }
1231
1232 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
1233 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1234 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
1235 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
1236 { \
1237 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1238 "atomic_cas_" #name "():args"); \
1239 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1240 "atomic_cas_" #name "():arg1"); \
1241 kmsan_init_ret(sizeof(tret)); \
1242 return atomic_cas_##name(ptr, exp, new); \
1243 } \
1244 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1245 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
1246 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
1247 { \
1248 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \
1249 "atomic_cas_" #name "_ni():args"); \
1250 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1251 "atomic_cas_" #name "_ni():arg1"); \
1252 kmsan_init_ret(sizeof(tret)); \
1253 return atomic_cas_##name##_ni(ptr, exp, new); \
1254 }
1255
1256 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
1257 tret atomic_swap_##name(volatile targ1 *, targ2); \
1258 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \
1259 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
1260 { \
1261 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1262 "atomic_swap_" #name "():args"); \
1263 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1264 "atomic_swap_" #name "():arg1"); \
1265 kmsan_init_ret(sizeof(tret)); \
1266 return atomic_swap_##name(ptr, val); \
1267 }
1268
1269 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
1270 void atomic_dec_##name(volatile targ1 *); \
1271 void kmsan_atomic_dec_##name(volatile targ1 *); \
1272 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \
1273 { \
1274 kmsan_check_arg(sizeof(ptr), \
1275 "atomic_dec_" #name "():args"); \
1276 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1277 "atomic_dec_" #name "():arg1"); \
1278 atomic_dec_##name(ptr); \
1279 } \
1280 tret atomic_dec_##name##_nv(volatile targ1 *); \
1281 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \
1282 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
1283 { \
1284 kmsan_check_arg(sizeof(ptr), \
1285 "atomic_dec_" #name "_nv():args"); \
1286 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1287 "atomic_dec_" #name "_nv():arg1"); \
1288 kmsan_init_ret(sizeof(tret)); \
1289 return atomic_dec_##name##_nv(ptr); \
1290 }
1291
1292 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
1293 void atomic_inc_##name(volatile targ1 *); \
1294 void kmsan_atomic_inc_##name(volatile targ1 *); \
1295 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \
1296 { \
1297 kmsan_check_arg(sizeof(ptr), \
1298 "atomic_inc_" #name "():args"); \
1299 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1300 "atomic_inc_" #name "():arg1"); \
1301 atomic_inc_##name(ptr); \
1302 } \
1303 tret atomic_inc_##name##_nv(volatile targ1 *); \
1304 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \
1305 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
1306 { \
1307 kmsan_check_arg(sizeof(ptr), \
1308 "atomic_inc_" #name "_nv():args"); \
1309 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \
1310 "atomic_inc_" #name "_nv():arg1"); \
1311 kmsan_init_ret(sizeof(tret)); \
1312 return atomic_inc_##name##_nv(ptr); \
1313 }
1314
1315 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
1316 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
1317 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
1318 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
1319 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
1320
1321 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
1322 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
1323 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
1324 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
1325
1326 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
1327 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
1328 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
1329 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
1330
1331 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
1332 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
1333 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
1334 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
1335 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
1336
1337 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
1338 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
1339 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
1340 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
1341 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
1342
1343 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
1344 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
1345 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
1346 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
1347 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
1348
1349 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
1350 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
1351 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
1352 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
1353 MSAN_ATOMIC_FUNC_INC(ptr, void *, void);
1354
1355 /* -------------------------------------------------------------------------- */
1356
1357 #include <sys/bus.h>
1358
1359 #undef bus_space_read_multi_1
1360 #undef bus_space_read_multi_2
1361 #undef bus_space_read_multi_4
1362 #undef bus_space_read_multi_8
1363 #undef bus_space_read_multi_stream_1
1364 #undef bus_space_read_multi_stream_2
1365 #undef bus_space_read_multi_stream_4
1366 #undef bus_space_read_multi_stream_8
1367 #undef bus_space_read_region_1
1368 #undef bus_space_read_region_2
1369 #undef bus_space_read_region_4
1370 #undef bus_space_read_region_8
1371 #undef bus_space_read_region_stream_1
1372 #undef bus_space_read_region_stream_2
1373 #undef bus_space_read_region_stream_4
1374 #undef bus_space_read_region_stream_8
1375
1376 #define MSAN_BUS_READ_FUNC(bytes, bits) \
1377 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1378 bus_size_t, uint##bits##_t *, bus_size_t); \
1379 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
1380 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1381 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
1382 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1383 bus_size_t count) \
1384 { \
1385 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1386 sizeof(uint##bits##_t) * count); \
1387 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
1388 } \
1389 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1390 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1391 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
1392 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1393 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
1394 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1395 bus_size_t count) \
1396 { \
1397 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1398 sizeof(uint##bits##_t) * count); \
1399 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
1400 } \
1401 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
1402 bus_size_t, uint##bits##_t *, bus_size_t); \
1403 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \
1404 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1405 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
1406 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1407 bus_size_t count) \
1408 { \
1409 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1410 sizeof(uint##bits##_t) * count); \
1411 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
1412 } \
1413 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1414 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1415 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
1416 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
1417 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
1418 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1419 bus_size_t count) \
1420 { \
1421 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \
1422 sizeof(uint##bits##_t) * count); \
1423 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1424 }
1425
1426 MSAN_BUS_READ_FUNC(1, 8)
1427 MSAN_BUS_READ_FUNC(2, 16)
1428 MSAN_BUS_READ_FUNC(4, 32)
1429 MSAN_BUS_READ_FUNC(8, 64)
1430
1431 #undef bus_space_write_multi_1
1432 #undef bus_space_write_multi_2
1433 #undef bus_space_write_multi_4
1434 #undef bus_space_write_multi_8
1435 #undef bus_space_write_multi_stream_1
1436 #undef bus_space_write_multi_stream_2
1437 #undef bus_space_write_multi_stream_4
1438 #undef bus_space_write_multi_stream_8
1439 #undef bus_space_write_region_1
1440 #undef bus_space_write_region_2
1441 #undef bus_space_write_region_4
1442 #undef bus_space_write_region_8
1443 #undef bus_space_write_region_stream_1
1444 #undef bus_space_write_region_stream_2
1445 #undef bus_space_write_region_stream_4
1446 #undef bus_space_write_region_stream_8
1447
1448 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \
1449 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1450 bus_size_t, const uint##bits##_t *, bus_size_t); \
1451 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1452 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1453 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1454 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1455 bus_size_t count) \
1456 { \
1457 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1458 "bus_space_write()"); \
1459 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1460 } \
1461 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1462 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1463 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1464 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1465 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1466 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1467 bus_size_t count) \
1468 { \
1469 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1470 "bus_space_write()"); \
1471 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1472 } \
1473 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1474 bus_size_t, const uint##bits##_t *, bus_size_t); \
1475 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \
1476 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1477 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1478 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1479 bus_size_t count) \
1480 { \
1481 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1482 "bus_space_write()"); \
1483 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1484 } \
1485 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1486 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1487 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1488 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1489 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1490 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1491 bus_size_t count) \
1492 { \
1493 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \
1494 "bus_space_write()"); \
1495 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1496 }
1497
1498 MSAN_BUS_WRITE_FUNC(1, 8)
1499 MSAN_BUS_WRITE_FUNC(2, 16)
1500 MSAN_BUS_WRITE_FUNC(4, 32)
1501 MSAN_BUS_WRITE_FUNC(8, 64)
1502
1503 /* -------------------------------------------------------------------------- */
1504
1505 #include <sys/mbuf.h>
1506
1507 static void
kmsan_dma_sync_linear(uint8_t * buf,bus_addr_t offset,bus_size_t len,bool init,uintptr_t pc)1508 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1509 bool init, uintptr_t pc)
1510 {
1511 if (init) {
1512 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len);
1513 } else {
1514 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp");
1515 }
1516 }
1517
1518 static void
kmsan_dma_sync_mbuf(struct mbuf * m,bus_addr_t offset,bus_size_t len,bool init,uintptr_t pc)1519 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1520 bool init, uintptr_t pc)
1521 {
1522 bus_addr_t minlen;
1523
1524 for (; m != NULL && len != 0; m = m->m_next) {
1525 if (offset >= m->m_len) {
1526 offset -= m->m_len;
1527 continue;
1528 }
1529
1530 minlen = MIN(len, m->m_len - offset);
1531
1532 if (init) {
1533 kmsan_shadow_fill(mtod(m, char *) + offset,
1534 KMSAN_STATE_INITED, minlen);
1535 } else {
1536 kmsan_shadow_check(mtod(m, char *) + offset,
1537 minlen, "MbufDmaSyncOp");
1538 }
1539
1540 offset = 0;
1541 len -= minlen;
1542 }
1543 }
1544
1545 static void
kmsan_dma_sync_uio(struct uio * uio,bus_addr_t offset,bus_size_t len,bool init,uintptr_t pc)1546 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1547 bool init, uintptr_t pc)
1548 {
1549 bus_size_t minlen, resid;
1550 struct iovec *iov;
1551 int i;
1552
1553 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
1554 return;
1555
1556 resid = uio->uio_resid;
1557 iov = uio->uio_iov;
1558
1559 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1560 minlen = MIN(resid, iov[i].iov_len);
1561
1562 if (init) {
1563 kmsan_shadow_fill(iov[i].iov_base,
1564 KMSAN_STATE_INITED, minlen);
1565 } else {
1566 kmsan_shadow_check(iov[i].iov_base, minlen,
1567 "UioDmaSyncOp");
1568 }
1569
1570 resid -= minlen;
1571 }
1572 }
1573
1574 void
kmsan_dma_sync(bus_dmamap_t map,bus_addr_t offset,bus_size_t len,int ops)1575 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1576 {
1577 bool init;
1578
1579 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0)
1580 return;
1581 init = (ops & BUS_DMASYNC_POSTREAD) != 0;
1582
1583 switch (map->dm_buftype) {
1584 case KMSAN_DMA_LINEAR:
1585 kmsan_dma_sync_linear(map->dm_buf, offset, len, init,
1586 __RET_ADDR);
1587 break;
1588 case KMSAN_DMA_MBUF:
1589 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init,
1590 __RET_ADDR);
1591 break;
1592 case KMSAN_DMA_UIO:
1593 kmsan_dma_sync_uio(map->dm_buf, offset, len, init,
1594 __RET_ADDR);
1595 break;
1596 case KMSAN_DMA_RAW:
1597 break;
1598 default:
1599 panic("%s: impossible", __func__);
1600 }
1601 }
1602
1603 void
kmsan_dma_load(bus_dmamap_t map,void * buf,bus_size_t buflen,int type)1604 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1605 {
1606 map->dm_buf = buf;
1607 map->dm_buflen = buflen;
1608 map->dm_buftype = type;
1609 }
1610