1 /* $NetBSD: subr_copy.c,v 1.19 2023/05/22 14:07:24 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 1997, 1998, 1999, 2002, 2007, 2008, 2019
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10 * NASA Ames Research Center.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 /*
35 * Copyright (c) 1982, 1986, 1991, 1993
36 * The Regents of the University of California. All rights reserved.
37 * (c) UNIX System Laboratories, Inc.
38 * All or some portions of this file are derived from material licensed
39 * to the University of California by American Telephone and Telegraph
40 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
41 * the permission of UNIX System Laboratories, Inc.
42 *
43 * Copyright (c) 1992, 1993
44 * The Regents of the University of California. All rights reserved.
45 *
46 * This software was developed by the Computer Systems Engineering group
47 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
48 * contributed to Berkeley.
49 *
50 * All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Lawrence Berkeley Laboratory.
54 *
55 * Redistribution and use in source and binary forms, with or without
56 * modification, are permitted provided that the following conditions
57 * are met:
58 * 1. Redistributions of source code must retain the above copyright
59 * notice, this list of conditions and the following disclaimer.
60 * 2. Redistributions in binary form must reproduce the above copyright
61 * notice, this list of conditions and the following disclaimer in the
62 * documentation and/or other materials provided with the distribution.
63 * 3. Neither the name of the University nor the names of its contributors
64 * may be used to endorse or promote products derived from this software
65 * without specific prior written permission.
66 *
67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
77 * SUCH DAMAGE.
78 *
79 * @(#)kern_subr.c 8.4 (Berkeley) 2/14/95
80 */
81
82 #include <sys/cdefs.h>
83 __KERNEL_RCSID(0, "$NetBSD: subr_copy.c,v 1.19 2023/05/22 14:07:24 riastradh Exp $");
84
85 #define __UFETCHSTORE_PRIVATE
86 #define __UCAS_PRIVATE
87
88 #include <sys/param.h>
89 #include <sys/fcntl.h>
90 #include <sys/proc.h>
91 #include <sys/systm.h>
92
93 #include <uvm/uvm_extern.h>
94
95 void
uio_setup_sysspace(struct uio * uio)96 uio_setup_sysspace(struct uio *uio)
97 {
98
99 uio->uio_vmspace = vmspace_kernel();
100 }
101
102 int
uiomove(void * buf,size_t n,struct uio * uio)103 uiomove(void *buf, size_t n, struct uio *uio)
104 {
105 struct vmspace *vm = uio->uio_vmspace;
106 struct iovec *iov;
107 size_t cnt;
108 int error = 0;
109 char *cp = buf;
110
111 ASSERT_SLEEPABLE();
112
113 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE);
114 while (n > 0 && uio->uio_resid) {
115 KASSERT(uio->uio_iovcnt > 0);
116 iov = uio->uio_iov;
117 cnt = iov->iov_len;
118 if (cnt == 0) {
119 KASSERT(uio->uio_iovcnt > 1);
120 uio->uio_iov++;
121 uio->uio_iovcnt--;
122 continue;
123 }
124 if (cnt > n)
125 cnt = n;
126 if (!VMSPACE_IS_KERNEL_P(vm)) {
127 preempt_point();
128 }
129
130 if (uio->uio_rw == UIO_READ) {
131 error = copyout_vmspace(vm, cp, iov->iov_base,
132 cnt);
133 } else {
134 error = copyin_vmspace(vm, iov->iov_base, cp,
135 cnt);
136 }
137 if (error) {
138 break;
139 }
140 iov->iov_base = (char *)iov->iov_base + cnt;
141 iov->iov_len -= cnt;
142 uio->uio_resid -= cnt;
143 uio->uio_offset += cnt;
144 cp += cnt;
145 KDASSERT(cnt <= n);
146 n -= cnt;
147 }
148
149 return (error);
150 }
151
152 /*
153 * Wrapper for uiomove() that validates the arguments against a known-good
154 * kernel buffer.
155 */
156 int
uiomove_frombuf(void * buf,size_t buflen,struct uio * uio)157 uiomove_frombuf(void *buf, size_t buflen, struct uio *uio)
158 {
159 size_t offset;
160
161 if (uio->uio_offset < 0 || /* uio->uio_resid < 0 || */
162 (offset = uio->uio_offset) != uio->uio_offset)
163 return (EINVAL);
164 if (offset >= buflen)
165 return (0);
166 return (uiomove((char *)buf + offset, buflen - offset, uio));
167 }
168
169 int
uiopeek(void * buf,size_t n,struct uio * uio)170 uiopeek(void *buf, size_t n, struct uio *uio)
171 {
172 struct vmspace *vm = uio->uio_vmspace;
173 struct iovec *iov;
174 size_t cnt;
175 int error = 0;
176 char *cp = buf;
177 size_t resid = uio->uio_resid;
178 int iovcnt = uio->uio_iovcnt;
179 char *base;
180 size_t len;
181
182 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE);
183
184 if (n == 0 || resid == 0)
185 return 0;
186 iov = uio->uio_iov;
187 base = iov->iov_base;
188 len = iov->iov_len;
189
190 while (n > 0 && resid > 0) {
191 KASSERT(iovcnt > 0);
192 cnt = len;
193 if (cnt == 0) {
194 KASSERT(iovcnt > 1);
195 iov++;
196 iovcnt--;
197 base = iov->iov_base;
198 len = iov->iov_len;
199 continue;
200 }
201 if (cnt > n)
202 cnt = n;
203 if (!VMSPACE_IS_KERNEL_P(vm)) {
204 preempt_point();
205 }
206
207 if (uio->uio_rw == UIO_READ) {
208 error = copyout_vmspace(vm, cp, base, cnt);
209 } else {
210 error = copyin_vmspace(vm, base, cp, cnt);
211 }
212 if (error) {
213 break;
214 }
215 base += cnt;
216 len -= cnt;
217 resid -= cnt;
218 cp += cnt;
219 KDASSERT(cnt <= n);
220 n -= cnt;
221 }
222
223 return error;
224 }
225
226 void
uioskip(size_t n,struct uio * uio)227 uioskip(size_t n, struct uio *uio)
228 {
229 struct iovec *iov;
230 size_t cnt;
231
232 KASSERTMSG(n <= uio->uio_resid, "n=%zu resid=%zu", n, uio->uio_resid);
233
234 KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE);
235 while (n > 0 && uio->uio_resid) {
236 KASSERT(uio->uio_iovcnt > 0);
237 iov = uio->uio_iov;
238 cnt = iov->iov_len;
239 if (cnt == 0) {
240 KASSERT(uio->uio_iovcnt > 1);
241 uio->uio_iov++;
242 uio->uio_iovcnt--;
243 continue;
244 }
245 if (cnt > n)
246 cnt = n;
247 iov->iov_base = (char *)iov->iov_base + cnt;
248 iov->iov_len -= cnt;
249 uio->uio_resid -= cnt;
250 uio->uio_offset += cnt;
251 KDASSERT(cnt <= n);
252 n -= cnt;
253 }
254 }
255
256 /*
257 * Give next character to user as result of read.
258 */
259 int
ureadc(int c,struct uio * uio)260 ureadc(int c, struct uio *uio)
261 {
262 struct iovec *iov;
263
264 if (uio->uio_resid <= 0)
265 panic("ureadc: non-positive resid");
266 again:
267 if (uio->uio_iovcnt <= 0)
268 panic("ureadc: non-positive iovcnt");
269 iov = uio->uio_iov;
270 if (iov->iov_len <= 0) {
271 uio->uio_iovcnt--;
272 uio->uio_iov++;
273 goto again;
274 }
275 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) {
276 int error;
277 if ((error = ustore_char(iov->iov_base, c)) != 0)
278 return (error);
279 } else {
280 *(char *)iov->iov_base = c;
281 }
282 iov->iov_base = (char *)iov->iov_base + 1;
283 iov->iov_len--;
284 uio->uio_resid--;
285 uio->uio_offset++;
286 return (0);
287 }
288
289 /*
290 * Like copyin(), but operates on an arbitrary vmspace.
291 */
292 int
copyin_vmspace(struct vmspace * vm,const void * uaddr,void * kaddr,size_t len)293 copyin_vmspace(struct vmspace *vm, const void *uaddr, void *kaddr, size_t len)
294 {
295 struct iovec iov;
296 struct uio uio;
297 int error;
298
299 if (len == 0)
300 return (0);
301
302 if (VMSPACE_IS_KERNEL_P(vm)) {
303 return kcopy(uaddr, kaddr, len);
304 }
305 if (__predict_true(vm == curproc->p_vmspace)) {
306 return copyin(uaddr, kaddr, len);
307 }
308
309 iov.iov_base = kaddr;
310 iov.iov_len = len;
311 uio.uio_iov = &iov;
312 uio.uio_iovcnt = 1;
313 uio.uio_offset = (off_t)(uintptr_t)uaddr;
314 uio.uio_resid = len;
315 uio.uio_rw = UIO_READ;
316 UIO_SETUP_SYSSPACE(&uio);
317 error = uvm_io(&vm->vm_map, &uio, 0);
318
319 return (error);
320 }
321
322 /*
323 * Like copyout(), but operates on an arbitrary vmspace.
324 */
325 int
copyout_vmspace(struct vmspace * vm,const void * kaddr,void * uaddr,size_t len)326 copyout_vmspace(struct vmspace *vm, const void *kaddr, void *uaddr, size_t len)
327 {
328 struct iovec iov;
329 struct uio uio;
330 int error;
331
332 if (len == 0)
333 return (0);
334
335 if (VMSPACE_IS_KERNEL_P(vm)) {
336 return kcopy(kaddr, uaddr, len);
337 }
338 if (__predict_true(vm == curproc->p_vmspace)) {
339 return copyout(kaddr, uaddr, len);
340 }
341
342 iov.iov_base = __UNCONST(kaddr); /* XXXUNCONST cast away const */
343 iov.iov_len = len;
344 uio.uio_iov = &iov;
345 uio.uio_iovcnt = 1;
346 uio.uio_offset = (off_t)(uintptr_t)uaddr;
347 uio.uio_resid = len;
348 uio.uio_rw = UIO_WRITE;
349 UIO_SETUP_SYSSPACE(&uio);
350 error = uvm_io(&vm->vm_map, &uio, 0);
351
352 return (error);
353 }
354
355 /*
356 * Like copyin(), but operates on an arbitrary process.
357 */
358 int
copyin_proc(struct proc * p,const void * uaddr,void * kaddr,size_t len)359 copyin_proc(struct proc *p, const void *uaddr, void *kaddr, size_t len)
360 {
361 struct vmspace *vm;
362 int error;
363
364 error = proc_vmspace_getref(p, &vm);
365 if (error) {
366 return error;
367 }
368 error = copyin_vmspace(vm, uaddr, kaddr, len);
369 uvmspace_free(vm);
370
371 return error;
372 }
373
374 /*
375 * Like copyout(), but operates on an arbitrary process.
376 */
377 int
copyout_proc(struct proc * p,const void * kaddr,void * uaddr,size_t len)378 copyout_proc(struct proc *p, const void *kaddr, void *uaddr, size_t len)
379 {
380 struct vmspace *vm;
381 int error;
382
383 error = proc_vmspace_getref(p, &vm);
384 if (error) {
385 return error;
386 }
387 error = copyout_vmspace(vm, kaddr, uaddr, len);
388 uvmspace_free(vm);
389
390 return error;
391 }
392
393 /*
394 * Like copyin(), but operates on an arbitrary pid.
395 */
396 int
copyin_pid(pid_t pid,const void * uaddr,void * kaddr,size_t len)397 copyin_pid(pid_t pid, const void *uaddr, void *kaddr, size_t len)
398 {
399 struct proc *p;
400 struct vmspace *vm;
401 int error;
402
403 mutex_enter(&proc_lock);
404 p = proc_find(pid);
405 if (p == NULL) {
406 mutex_exit(&proc_lock);
407 return ESRCH;
408 }
409 mutex_enter(p->p_lock);
410 error = proc_vmspace_getref(p, &vm);
411 mutex_exit(p->p_lock);
412 mutex_exit(&proc_lock);
413
414 if (error == 0) {
415 error = copyin_vmspace(vm, uaddr, kaddr, len);
416 uvmspace_free(vm);
417 }
418 return error;
419 }
420
421 /*
422 * Like copyin(), except it operates on kernel addresses when the FKIOCTL
423 * flag is passed in `ioctlflags' from the ioctl call.
424 */
425 int
ioctl_copyin(int ioctlflags,const void * src,void * dst,size_t len)426 ioctl_copyin(int ioctlflags, const void *src, void *dst, size_t len)
427 {
428 if (ioctlflags & FKIOCTL)
429 return kcopy(src, dst, len);
430 return copyin(src, dst, len);
431 }
432
433 /*
434 * Like copyout(), except it operates on kernel addresses when the FKIOCTL
435 * flag is passed in `ioctlflags' from the ioctl call.
436 */
437 int
ioctl_copyout(int ioctlflags,const void * src,void * dst,size_t len)438 ioctl_copyout(int ioctlflags, const void *src, void *dst, size_t len)
439 {
440 if (ioctlflags & FKIOCTL)
441 return kcopy(src, dst, len);
442 return copyout(src, dst, len);
443 }
444
445 /*
446 * User-space CAS / fetch / store
447 */
448
449 #ifdef __NO_STRICT_ALIGNMENT
450 #define CHECK_ALIGNMENT(x) __nothing
451 #else /* ! __NO_STRICT_ALIGNMENT */
452 static bool
ufetchstore_aligned(uintptr_t uaddr,size_t size)453 ufetchstore_aligned(uintptr_t uaddr, size_t size)
454 {
455 return (uaddr & (size - 1)) == 0;
456 }
457
458 #define CHECK_ALIGNMENT() \
459 do { \
460 if (!ufetchstore_aligned((uintptr_t)uaddr, sizeof(*uaddr))) \
461 return EFAULT; \
462 } while (/*CONSTCOND*/0)
463 #endif /* __NO_STRICT_ALIGNMENT */
464
465 /*
466 * __HAVE_UCAS_FULL platforms provide _ucas_32() and _ucas_64() themselves.
467 * _RUMPKERNEL also provides it's own _ucas_32() and _ucas_64().
468 *
469 * In all other cases, we provide generic implementations that work on
470 * all platforms.
471 */
472
473 #if !defined(__HAVE_UCAS_FULL) && !defined(_RUMPKERNEL)
474 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)
475 #include <sys/atomic.h>
476 #include <sys/cpu.h>
477 #include <sys/once.h>
478 #include <sys/mutex.h>
479 #include <sys/ipi.h>
480
481 static int ucas_critical_splcookie;
482 static volatile u_int ucas_critical_pausing_cpus;
483 static u_int ucas_critical_ipi;
ONCE_DECL(ucas_critical_init_once)484 static ONCE_DECL(ucas_critical_init_once)
485
486 static void
487 ucas_critical_cpu_gate(void *arg __unused)
488 {
489 int count = SPINLOCK_BACKOFF_MIN;
490
491 KASSERT(atomic_load_relaxed(&ucas_critical_pausing_cpus) > 0);
492
493 /*
494 * Notify ucas_critical_wait that we have stopped. Using
495 * store-release ensures all our memory operations up to the
496 * IPI happen before the ucas -- no buffered stores on our end
497 * can clobber it later on, for instance.
498 *
499 * Matches atomic_load_acquire in ucas_critical_wait -- turns
500 * the following atomic_dec_uint into a store-release.
501 */
502 membar_release();
503 atomic_dec_uint(&ucas_critical_pausing_cpus);
504
505 /*
506 * Wait for ucas_critical_exit to reopen the gate and let us
507 * proceed. Using a load-acquire ensures the ucas happens
508 * before any of our memory operations when we return from the
509 * IPI and proceed -- we won't observe any stale cached value
510 * that the ucas overwrote, for instance.
511 *
512 * Matches atomic_store_release in ucas_critical_exit.
513 */
514 while (atomic_load_acquire(&ucas_critical_pausing_cpus) != (u_int)-1) {
515 SPINLOCK_BACKOFF(count);
516 }
517 }
518
519 static int
ucas_critical_init(void)520 ucas_critical_init(void)
521 {
522
523 ucas_critical_ipi = ipi_register(ucas_critical_cpu_gate, NULL);
524 return 0;
525 }
526
527 static void
ucas_critical_wait(void)528 ucas_critical_wait(void)
529 {
530 int count = SPINLOCK_BACKOFF_MIN;
531
532 /*
533 * Wait for all CPUs to stop at the gate. Using a load-acquire
534 * ensures all memory operations before they stop at the gate
535 * happen before the ucas -- no buffered stores in other CPUs
536 * can clobber it later on, for instance.
537 *
538 * Matches membar_release/atomic_dec_uint (store-release) in
539 * ucas_critical_cpu_gate.
540 */
541 while (atomic_load_acquire(&ucas_critical_pausing_cpus) > 0) {
542 SPINLOCK_BACKOFF(count);
543 }
544 }
545 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */
546
547 static inline void
ucas_critical_enter(lwp_t * const l)548 ucas_critical_enter(lwp_t * const l)
549 {
550
551 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)
552 if (ncpu > 1) {
553 RUN_ONCE(&ucas_critical_init_once, ucas_critical_init);
554
555 /*
556 * Acquire the mutex first, then go to splhigh() and
557 * broadcast the IPI to lock all of the other CPUs
558 * behind the gate.
559 *
560 * N.B. Going to splhigh() implicitly disables preemption,
561 * so there's no need to do it explicitly.
562 */
563 mutex_enter(&cpu_lock);
564 ucas_critical_splcookie = splhigh();
565 ucas_critical_pausing_cpus = ncpu - 1;
566 ipi_trigger_broadcast(ucas_critical_ipi, true);
567 ucas_critical_wait();
568 return;
569 }
570 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */
571
572 KPREEMPT_DISABLE(l);
573 }
574
575 static inline void
ucas_critical_exit(lwp_t * const l)576 ucas_critical_exit(lwp_t * const l)
577 {
578
579 #if !defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)
580 if (ncpu > 1) {
581 /*
582 * Open the gate and notify all CPUs in
583 * ucas_critical_cpu_gate that they can now proceed.
584 * Using a store-release ensures the ucas happens
585 * before any memory operations they issue after the
586 * IPI -- they won't observe any stale cache of the
587 * target word, for instance.
588 *
589 * Matches atomic_load_acquire in ucas_critical_cpu_gate.
590 */
591 atomic_store_release(&ucas_critical_pausing_cpus, (u_int)-1);
592 splx(ucas_critical_splcookie);
593 mutex_exit(&cpu_lock);
594 return;
595 }
596 #endif /* ! __HAVE_UCAS_MP && MULTIPROCESSOR */
597
598 KPREEMPT_ENABLE(l);
599 }
600
601 int
_ucas_32(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)602 _ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, uint32_t *ret)
603 {
604 lwp_t * const l = curlwp;
605 uint32_t *uva = ((void *)(uintptr_t)uaddr);
606 int error;
607
608 /*
609 * Wire the user address down to avoid taking a page fault during
610 * the critical section.
611 */
612 error = uvm_vslock(l->l_proc->p_vmspace, uva, sizeof(*uaddr),
613 VM_PROT_READ | VM_PROT_WRITE);
614 if (error)
615 return error;
616
617 ucas_critical_enter(l);
618 error = _ufetch_32(uva, ret);
619 if (error == 0 && *ret == old) {
620 error = _ustore_32(uva, new);
621 }
622 ucas_critical_exit(l);
623
624 uvm_vsunlock(l->l_proc->p_vmspace, uva, sizeof(*uaddr));
625
626 return error;
627 }
628
629 #ifdef _LP64
630 int
_ucas_64(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)631 _ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, uint64_t *ret)
632 {
633 lwp_t * const l = curlwp;
634 uint64_t *uva = ((void *)(uintptr_t)uaddr);
635 int error;
636
637 /*
638 * Wire the user address down to avoid taking a page fault during
639 * the critical section.
640 */
641 error = uvm_vslock(l->l_proc->p_vmspace, uva, sizeof(*uaddr),
642 VM_PROT_READ | VM_PROT_WRITE);
643 if (error)
644 return error;
645
646 ucas_critical_enter(l);
647 error = _ufetch_64(uva, ret);
648 if (error == 0 && *ret == old) {
649 error = _ustore_64(uva, new);
650 }
651 ucas_critical_exit(l);
652
653 uvm_vsunlock(l->l_proc->p_vmspace, uva, sizeof(*uaddr));
654
655 return error;
656 }
657 #endif /* _LP64 */
658 #endif /* ! __HAVE_UCAS_FULL && ! _RUMPKERNEL */
659
660 int
ucas_32(volatile uint32_t * uaddr,uint32_t old,uint32_t new,uint32_t * ret)661 ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, uint32_t *ret)
662 {
663
664 ASSERT_SLEEPABLE();
665 CHECK_ALIGNMENT();
666 #if (defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)) && \
667 !defined(_RUMPKERNEL)
668 if (ncpu > 1) {
669 return _ucas_32_mp(uaddr, old, new, ret);
670 }
671 #endif /* __HAVE_UCAS_MP && MULTIPROCESSOR */
672 return _ucas_32(uaddr, old, new, ret);
673 }
674
675 #ifdef _LP64
676 int
ucas_64(volatile uint64_t * uaddr,uint64_t old,uint64_t new,uint64_t * ret)677 ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, uint64_t *ret)
678 {
679
680 ASSERT_SLEEPABLE();
681 CHECK_ALIGNMENT();
682 #if (defined(__HAVE_UCAS_MP) && defined(MULTIPROCESSOR)) && \
683 !defined(_RUMPKERNEL)
684 if (ncpu > 1) {
685 return _ucas_64_mp(uaddr, old, new, ret);
686 }
687 #endif /* __HAVE_UCAS_MP && MULTIPROCESSOR */
688 return _ucas_64(uaddr, old, new, ret);
689 }
690 #endif /* _LP64 */
691
692 __strong_alias(ucas_int,ucas_32);
693 #ifdef _LP64
694 __strong_alias(ucas_ptr,ucas_64);
695 #else
696 __strong_alias(ucas_ptr,ucas_32);
697 #endif /* _LP64 */
698
699 int
ufetch_8(const uint8_t * uaddr,uint8_t * valp)700 ufetch_8(const uint8_t *uaddr, uint8_t *valp)
701 {
702
703 ASSERT_SLEEPABLE();
704 CHECK_ALIGNMENT();
705 return _ufetch_8(uaddr, valp);
706 }
707
708 int
ufetch_16(const uint16_t * uaddr,uint16_t * valp)709 ufetch_16(const uint16_t *uaddr, uint16_t *valp)
710 {
711
712 ASSERT_SLEEPABLE();
713 CHECK_ALIGNMENT();
714 return _ufetch_16(uaddr, valp);
715 }
716
717 int
ufetch_32(const uint32_t * uaddr,uint32_t * valp)718 ufetch_32(const uint32_t *uaddr, uint32_t *valp)
719 {
720
721 ASSERT_SLEEPABLE();
722 CHECK_ALIGNMENT();
723 return _ufetch_32(uaddr, valp);
724 }
725
726 #ifdef _LP64
727 int
ufetch_64(const uint64_t * uaddr,uint64_t * valp)728 ufetch_64(const uint64_t *uaddr, uint64_t *valp)
729 {
730
731 ASSERT_SLEEPABLE();
732 CHECK_ALIGNMENT();
733 return _ufetch_64(uaddr, valp);
734 }
735 #endif /* _LP64 */
736
737 __strong_alias(ufetch_char,ufetch_8);
738 __strong_alias(ufetch_short,ufetch_16);
739 __strong_alias(ufetch_int,ufetch_32);
740 #ifdef _LP64
741 __strong_alias(ufetch_long,ufetch_64);
742 __strong_alias(ufetch_ptr,ufetch_64);
743 #else
744 __strong_alias(ufetch_long,ufetch_32);
745 __strong_alias(ufetch_ptr,ufetch_32);
746 #endif /* _LP64 */
747
748 int
ustore_8(uint8_t * uaddr,uint8_t val)749 ustore_8(uint8_t *uaddr, uint8_t val)
750 {
751
752 ASSERT_SLEEPABLE();
753 CHECK_ALIGNMENT();
754 return _ustore_8(uaddr, val);
755 }
756
757 int
ustore_16(uint16_t * uaddr,uint16_t val)758 ustore_16(uint16_t *uaddr, uint16_t val)
759 {
760
761 ASSERT_SLEEPABLE();
762 CHECK_ALIGNMENT();
763 return _ustore_16(uaddr, val);
764 }
765
766 int
ustore_32(uint32_t * uaddr,uint32_t val)767 ustore_32(uint32_t *uaddr, uint32_t val)
768 {
769
770 ASSERT_SLEEPABLE();
771 CHECK_ALIGNMENT();
772 return _ustore_32(uaddr, val);
773 }
774
775 #ifdef _LP64
776 int
ustore_64(uint64_t * uaddr,uint64_t val)777 ustore_64(uint64_t *uaddr, uint64_t val)
778 {
779
780 ASSERT_SLEEPABLE();
781 CHECK_ALIGNMENT();
782 return _ustore_64(uaddr, val);
783 }
784 #endif /* _LP64 */
785
786 __strong_alias(ustore_char,ustore_8);
787 __strong_alias(ustore_short,ustore_16);
788 __strong_alias(ustore_int,ustore_32);
789 #ifdef _LP64
790 __strong_alias(ustore_long,ustore_64);
791 __strong_alias(ustore_ptr,ustore_64);
792 #else
793 __strong_alias(ustore_long,ustore_32);
794 __strong_alias(ustore_ptr,ustore_32);
795 #endif /* _LP64 */
796