1 /* $NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KCSAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $");
33
34 #include <sys/param.h>
35 #include <sys/device.h>
36 #include <sys/kernel.h>
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/systm.h>
40 #include <sys/types.h>
41 #include <sys/csan.h>
42 #include <sys/cpu.h>
43 #include <sys/pserialize.h>
44
45 #ifdef KCSAN_PANIC
46 #define REPORT panic
47 #else
48 #define REPORT printf
49 #endif
50
51 typedef struct {
52 uintptr_t addr;
53 uint32_t size;
54 bool write:1;
55 bool atomic:1;
56 uintptr_t pc;
57 } csan_cell_t;
58
59 typedef struct {
60 bool inited;
61 uint32_t cnt;
62 csan_cell_t cell;
63 } csan_cpu_t;
64
65 static csan_cpu_t kcsan_cpus[MAXCPUS];
66 static bool kcsan_enabled __read_mostly;
67
68 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
69
70 #define KCSAN_NACCESSES 1024
71 #define KCSAN_DELAY 10 /* 10 microseconds */
72
73 /* -------------------------------------------------------------------------- */
74
75 /* The MD code. */
76 #include <machine/csan.h>
77
78 /* -------------------------------------------------------------------------- */
79
80 void
kcsan_init(void)81 kcsan_init(void)
82 {
83 kcsan_enabled = true;
84 }
85
86 void
kcsan_cpu_init(struct cpu_info * ci)87 kcsan_cpu_init(struct cpu_info *ci)
88 {
89 kcsan_cpus[cpu_index(ci)].inited = true;
90 }
91
92 /* -------------------------------------------------------------------------- */
93
94 static inline void
kcsan_report(csan_cell_t * new,cpuid_t newcpu,csan_cell_t * old,cpuid_t oldcpu)95 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
96 {
97 const char *newsym, *oldsym;
98 int s;
99
100 s = pserialize_read_enter();
101 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
102 newsym = "Unknown";
103 }
104 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
105 oldsym = "Unknown";
106 }
107 REPORT("CSan: Racy Access "
108 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
109 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
110 newcpu,
111 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
112 (void *)new->addr, new->size, (void *)new->pc, newsym,
113 oldcpu,
114 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
115 (void *)old->addr, old->size, (void *)old->pc, oldsym);
116 pserialize_read_exit(s);
117 kcsan_md_unwind();
118 }
119
120 static inline bool
kcsan_access_is_atomic(csan_cell_t * new,csan_cell_t * old)121 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
122 {
123 if (new->write && !new->atomic)
124 return false;
125 if (old->write && !old->atomic)
126 return false;
127 return true;
128 }
129
130 static inline void
kcsan_access(uintptr_t addr,size_t size,bool write,bool atomic,uintptr_t pc)131 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
132 {
133 csan_cell_t old, new;
134 csan_cpu_t *cpu;
135 uint64_t intr;
136 size_t i;
137
138 if (__predict_false(!kcsan_enabled))
139 return;
140 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr)))
141 return;
142
143 new.addr = addr;
144 new.size = size;
145 new.write = write;
146 new.atomic = atomic;
147 new.pc = pc;
148
149 for (i = 0; i < ncpu; i++) {
150 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
151
152 if (old.addr + old.size <= new.addr)
153 continue;
154 if (new.addr + new.size <= old.addr)
155 continue;
156 if (__predict_true(!old.write && !new.write))
157 continue;
158 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
159 continue;
160
161 kcsan_report(&new, cpu_number(), &old, i);
162 break;
163 }
164
165 if (__predict_false(!kcsan_md_is_avail()))
166 return;
167
168 kcsan_md_disable_intrs(&intr);
169
170 cpu = &kcsan_cpus[cpu_number()];
171 if (__predict_false(!cpu->inited))
172 goto out;
173 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
174 if (__predict_true(cpu->cnt != 0))
175 goto out;
176
177 __builtin_memcpy(&cpu->cell, &new, sizeof(new));
178 kcsan_md_delay(KCSAN_DELAY);
179 __builtin_memset(&cpu->cell, 0, sizeof(new));
180
181 out:
182 kcsan_md_enable_intrs(&intr);
183 }
184
185 #define CSAN_READ(size) \
186 void __tsan_read##size(uintptr_t); \
187 void __tsan_read##size(uintptr_t addr) \
188 { \
189 kcsan_access(addr, size, false, false, __RET_ADDR); \
190 }
191
192 CSAN_READ(1)
193 CSAN_READ(2)
194 CSAN_READ(4)
195 CSAN_READ(8)
196 CSAN_READ(16)
197
198 #define CSAN_WRITE(size) \
199 void __tsan_write##size(uintptr_t); \
200 void __tsan_write##size(uintptr_t addr) \
201 { \
202 kcsan_access(addr, size, true, false, __RET_ADDR); \
203 }
204
205 CSAN_WRITE(1)
206 CSAN_WRITE(2)
207 CSAN_WRITE(4)
208 CSAN_WRITE(8)
209 CSAN_WRITE(16)
210
211 void __tsan_read_range(uintptr_t, size_t);
212 void __tsan_write_range(uintptr_t, size_t);
213
214 void
__tsan_read_range(uintptr_t addr,size_t size)215 __tsan_read_range(uintptr_t addr, size_t size)
216 {
217 kcsan_access(addr, size, false, false, __RET_ADDR);
218 }
219
220 void
__tsan_write_range(uintptr_t addr,size_t size)221 __tsan_write_range(uintptr_t addr, size_t size)
222 {
223 kcsan_access(addr, size, true, false, __RET_ADDR);
224 }
225
226 void __tsan_init(void);
227 void __tsan_func_entry(void *);
228 void __tsan_func_exit(void);
229
230 void
__tsan_init(void)231 __tsan_init(void)
232 {
233 }
234
235 void
__tsan_func_entry(void * call_pc)236 __tsan_func_entry(void *call_pc)
237 {
238 }
239
240 void
__tsan_func_exit(void)241 __tsan_func_exit(void)
242 {
243 }
244
245 /* -------------------------------------------------------------------------- */
246
247 void *
kcsan_memcpy(void * dst,const void * src,size_t len)248 kcsan_memcpy(void *dst, const void *src, size_t len)
249 {
250 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
251 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
252 return __builtin_memcpy(dst, src, len);
253 }
254
255 int
kcsan_memcmp(const void * b1,const void * b2,size_t len)256 kcsan_memcmp(const void *b1, const void *b2, size_t len)
257 {
258 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
259 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
260 return __builtin_memcmp(b1, b2, len);
261 }
262
263 void *
kcsan_memset(void * b,int c,size_t len)264 kcsan_memset(void *b, int c, size_t len)
265 {
266 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
267 return __builtin_memset(b, c, len);
268 }
269
270 void *
kcsan_memmove(void * dst,const void * src,size_t len)271 kcsan_memmove(void *dst, const void *src, size_t len)
272 {
273 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
274 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
275 return __builtin_memmove(dst, src, len);
276 }
277
278 char *
kcsan_strcpy(char * dst,const char * src)279 kcsan_strcpy(char *dst, const char *src)
280 {
281 char *save = dst;
282
283 while (1) {
284 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
285 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
286 *dst = *src;
287 if (*src == '\0')
288 break;
289 src++, dst++;
290 }
291
292 return save;
293 }
294
295 int
kcsan_strcmp(const char * s1,const char * s2)296 kcsan_strcmp(const char *s1, const char *s2)
297 {
298 while (1) {
299 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
300 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
301 if (*s1 != *s2)
302 break;
303 if (*s1 == '\0')
304 return 0;
305 s1++, s2++;
306 }
307
308 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
309 }
310
311 size_t
kcsan_strlen(const char * str)312 kcsan_strlen(const char *str)
313 {
314 const char *s;
315
316 s = str;
317 while (1) {
318 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
319 if (*s == '\0')
320 break;
321 s++;
322 }
323
324 return (s - str);
325 }
326
327 #undef kcopy
328 #undef copyinstr
329 #undef copyoutstr
330 #undef copyin
331 #undef copyout
332
333 int kcsan_kcopy(const void *, void *, size_t);
334 int kcsan_copyinstr(const void *, void *, size_t, size_t *);
335 int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
336 int kcsan_copyin(const void *, void *, size_t);
337 int kcsan_copyout(const void *, void *, size_t);
338 int kcopy(const void *, void *, size_t);
339 int copyinstr(const void *, void *, size_t, size_t *);
340 int copyoutstr(const void *, void *, size_t, size_t *);
341 int copyin(const void *, void *, size_t);
342 int copyout(const void *, void *, size_t);
343
344 int
kcsan_kcopy(const void * src,void * dst,size_t len)345 kcsan_kcopy(const void *src, void *dst, size_t len)
346 {
347 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
348 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
349 return kcopy(src, dst, len);
350 }
351
352 int
kcsan_copyin(const void * uaddr,void * kaddr,size_t len)353 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
354 {
355 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
356 return copyin(uaddr, kaddr, len);
357 }
358
359 int
kcsan_copyout(const void * kaddr,void * uaddr,size_t len)360 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
361 {
362 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
363 return copyout(kaddr, uaddr, len);
364 }
365
366 int
kcsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)367 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
368 {
369 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
370 return copyinstr(uaddr, kaddr, len, done);
371 }
372
373 int
kcsan_copyoutstr(const void * kaddr,void * uaddr,size_t len,size_t * done)374 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
375 {
376 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
377 return copyoutstr(kaddr, uaddr, len, done);
378 }
379
380 /* -------------------------------------------------------------------------- */
381
382 #undef atomic_add_32
383 #undef atomic_add_int
384 #undef atomic_add_long
385 #undef atomic_add_ptr
386 #undef atomic_add_64
387 #undef atomic_add_32_nv
388 #undef atomic_add_int_nv
389 #undef atomic_add_long_nv
390 #undef atomic_add_ptr_nv
391 #undef atomic_add_64_nv
392 #undef atomic_and_32
393 #undef atomic_and_uint
394 #undef atomic_and_ulong
395 #undef atomic_and_64
396 #undef atomic_and_32_nv
397 #undef atomic_and_uint_nv
398 #undef atomic_and_ulong_nv
399 #undef atomic_and_64_nv
400 #undef atomic_or_32
401 #undef atomic_or_uint
402 #undef atomic_or_ulong
403 #undef atomic_or_64
404 #undef atomic_or_32_nv
405 #undef atomic_or_uint_nv
406 #undef atomic_or_ulong_nv
407 #undef atomic_or_64_nv
408 #undef atomic_cas_32
409 #undef atomic_cas_uint
410 #undef atomic_cas_ulong
411 #undef atomic_cas_ptr
412 #undef atomic_cas_64
413 #undef atomic_cas_32_ni
414 #undef atomic_cas_uint_ni
415 #undef atomic_cas_ulong_ni
416 #undef atomic_cas_ptr_ni
417 #undef atomic_cas_64_ni
418 #undef atomic_swap_32
419 #undef atomic_swap_uint
420 #undef atomic_swap_ulong
421 #undef atomic_swap_ptr
422 #undef atomic_swap_64
423 #undef atomic_dec_32
424 #undef atomic_dec_uint
425 #undef atomic_dec_ulong
426 #undef atomic_dec_ptr
427 #undef atomic_dec_64
428 #undef atomic_dec_32_nv
429 #undef atomic_dec_uint_nv
430 #undef atomic_dec_ulong_nv
431 #undef atomic_dec_ptr_nv
432 #undef atomic_dec_64_nv
433 #undef atomic_inc_32
434 #undef atomic_inc_uint
435 #undef atomic_inc_ulong
436 #undef atomic_inc_ptr
437 #undef atomic_inc_64
438 #undef atomic_inc_32_nv
439 #undef atomic_inc_uint_nv
440 #undef atomic_inc_ulong_nv
441 #undef atomic_inc_ptr_nv
442 #undef atomic_inc_64_nv
443
444 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
445 void atomic_add_##name(volatile targ1 *, targ2); \
446 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
447 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
448 { \
449 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
450 __RET_ADDR); \
451 atomic_add_##name(ptr, val); \
452 } \
453 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
454 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
455 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
456 { \
457 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
458 __RET_ADDR); \
459 return atomic_add_##name##_nv(ptr, val); \
460 }
461
462 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
463 void atomic_and_##name(volatile targ1 *, targ2); \
464 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
465 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
466 { \
467 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
468 __RET_ADDR); \
469 atomic_and_##name(ptr, val); \
470 } \
471 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
472 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
473 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
474 { \
475 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
476 __RET_ADDR); \
477 return atomic_and_##name##_nv(ptr, val); \
478 }
479
480 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
481 void atomic_or_##name(volatile targ1 *, targ2); \
482 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
483 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
484 { \
485 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
486 __RET_ADDR); \
487 atomic_or_##name(ptr, val); \
488 } \
489 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
490 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
491 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
492 { \
493 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
494 __RET_ADDR); \
495 return atomic_or_##name##_nv(ptr, val); \
496 }
497
498 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
499 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
500 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
501 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
502 { \
503 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
504 __RET_ADDR); \
505 return atomic_cas_##name(ptr, exp, new); \
506 } \
507 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
508 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
509 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
510 { \
511 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
512 __RET_ADDR); \
513 return atomic_cas_##name##_ni(ptr, exp, new); \
514 }
515
516 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
517 tret atomic_swap_##name(volatile targ1 *, targ2); \
518 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
519 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
520 { \
521 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
522 __RET_ADDR); \
523 return atomic_swap_##name(ptr, val); \
524 }
525
526 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
527 void atomic_dec_##name(volatile targ1 *); \
528 void kcsan_atomic_dec_##name(volatile targ1 *); \
529 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
530 { \
531 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
532 __RET_ADDR); \
533 atomic_dec_##name(ptr); \
534 } \
535 tret atomic_dec_##name##_nv(volatile targ1 *); \
536 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \
537 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
538 { \
539 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
540 __RET_ADDR); \
541 return atomic_dec_##name##_nv(ptr); \
542 }
543
544 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
545 void atomic_inc_##name(volatile targ1 *); \
546 void kcsan_atomic_inc_##name(volatile targ1 *); \
547 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \
548 { \
549 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
550 __RET_ADDR); \
551 atomic_inc_##name(ptr); \
552 } \
553 tret atomic_inc_##name##_nv(volatile targ1 *); \
554 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \
555 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
556 { \
557 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
558 __RET_ADDR); \
559 return atomic_inc_##name##_nv(ptr); \
560 }
561
562 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
563 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
564 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
565 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
566 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
567
568 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
569 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
570 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
571 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
572
573 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
574 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
575 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
576 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
577
578 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
579 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
580 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
581 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
582 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
583
584 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
585 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
586 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
587 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
588 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
589
590 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
591 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
592 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
593 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
594 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
595
596 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
597 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
598 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
599 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
600 CSAN_ATOMIC_FUNC_INC(ptr, void *, void);
601
602 void
kcsan_atomic_load(const volatile void * p,void * v,int size)603 kcsan_atomic_load(const volatile void *p, void *v, int size)
604 {
605 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR);
606 switch (size) {
607 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break;
608 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break;
609 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break;
610 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break;
611 }
612 }
613
614 void
kcsan_atomic_store(volatile void * p,const void * v,int size)615 kcsan_atomic_store(volatile void *p, const void *v, int size)
616 {
617 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR);
618 #ifdef __HAVE_HASHLOCKED_ATOMICS
619 __do_atomic_store(p, v, size);
620 #else
621 switch (size) {
622 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break;
623 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break;
624 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break;
625 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break;
626 }
627 #endif
628 }
629
630 /* -------------------------------------------------------------------------- */
631
632 #include <sys/bus.h>
633
634 #undef bus_space_read_multi_1
635 #undef bus_space_read_multi_2
636 #undef bus_space_read_multi_4
637 #undef bus_space_read_multi_8
638 #undef bus_space_read_multi_stream_1
639 #undef bus_space_read_multi_stream_2
640 #undef bus_space_read_multi_stream_4
641 #undef bus_space_read_multi_stream_8
642 #undef bus_space_read_region_1
643 #undef bus_space_read_region_2
644 #undef bus_space_read_region_4
645 #undef bus_space_read_region_8
646 #undef bus_space_read_region_stream_1
647 #undef bus_space_read_region_stream_2
648 #undef bus_space_read_region_stream_4
649 #undef bus_space_read_region_stream_8
650 #undef bus_space_write_multi_1
651 #undef bus_space_write_multi_2
652 #undef bus_space_write_multi_4
653 #undef bus_space_write_multi_8
654 #undef bus_space_write_multi_stream_1
655 #undef bus_space_write_multi_stream_2
656 #undef bus_space_write_multi_stream_4
657 #undef bus_space_write_multi_stream_8
658 #undef bus_space_write_region_1
659 #undef bus_space_write_region_2
660 #undef bus_space_write_region_4
661 #undef bus_space_write_region_8
662 #undef bus_space_write_region_stream_1
663 #undef bus_space_write_region_stream_2
664 #undef bus_space_write_region_stream_4
665 #undef bus_space_write_region_stream_8
666
667 #define CSAN_BUS_READ_FUNC(bytes, bits) \
668 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
669 bus_size_t, uint##bits##_t *, bus_size_t); \
670 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
671 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
672 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
673 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
674 bus_size_t count) \
675 { \
676 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
677 false, false, __RET_ADDR); \
678 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
679 } \
680 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
681 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
682 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
683 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
684 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
685 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
686 bus_size_t count) \
687 { \
688 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
689 false, false, __RET_ADDR); \
690 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
691 } \
692 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
693 bus_size_t, uint##bits##_t *, bus_size_t); \
694 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \
695 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
696 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
697 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
698 bus_size_t count) \
699 { \
700 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
701 false, false, __RET_ADDR); \
702 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
703 } \
704 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
705 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
706 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
707 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
708 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
709 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
710 bus_size_t count) \
711 { \
712 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
713 false, false, __RET_ADDR); \
714 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
715 }
716
717 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \
718 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
719 bus_size_t, const uint##bits##_t *, bus_size_t); \
720 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
721 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
722 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
723 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
724 bus_size_t count) \
725 { \
726 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
727 true, false, __RET_ADDR); \
728 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
729 } \
730 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
731 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
732 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
733 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
734 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
735 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
736 bus_size_t count) \
737 { \
738 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
739 true, false, __RET_ADDR); \
740 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
741 } \
742 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
743 bus_size_t, const uint##bits##_t *, bus_size_t); \
744 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \
745 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
746 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
747 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
748 bus_size_t count) \
749 { \
750 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
751 true, false, __RET_ADDR); \
752 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
753 } \
754 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
755 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
756 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
757 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
758 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
759 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
760 bus_size_t count) \
761 { \
762 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
763 true, false, __RET_ADDR); \
764 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
765 }
766
767 CSAN_BUS_READ_FUNC(1, 8)
768 CSAN_BUS_READ_FUNC(2, 16)
769 CSAN_BUS_READ_FUNC(4, 32)
770 CSAN_BUS_READ_FUNC(8, 64)
771
772 CSAN_BUS_WRITE_FUNC(1, 8)
773 CSAN_BUS_WRITE_FUNC(2, 16)
774 CSAN_BUS_WRITE_FUNC(4, 32)
775 CSAN_BUS_WRITE_FUNC(8, 64)
776