1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/lock.h>
34 #include <sys/mutex.h>
35 #include <sys/pcpu.h>
36 #include <sys/proc.h>
37 #include <sys/sched.h>
38 #include <sys/sysctl.h>
39 #include <sys/systm.h>
40 #include <vm/vm.h>
41 #include <vm/vm_param.h>
42 #include <vm/vm_extern.h>
43 #include <vm/pmap.h>
44 #include <vm/vm_map.h>
45 #include <vm/vm_page.h>
46
47 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
48 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
49 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
50 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
51 int fubyte_fast(volatile const void *base, u_int kcr3);
52 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
53 int fuword16_fast(volatile const void *base, u_int kcr3);
54 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
55 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
56 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
57 int subyte_fast(volatile void *base, int val, u_int kcr3);
58 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
59 int suword16_fast(volatile void *base, int val, u_int kcr3);
60 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
61 int suword_fast(volatile void *base, long val, u_int kcr3);
62 static int (*suword_fast_tramp)(volatile void *, long, u_int);
63
64 static int fast_copyout = 1;
65 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
66 &fast_copyout, 0,
67 "");
68
69 void
copyout_init_tramp(void)70 copyout_init_tramp(void)
71 {
72
73 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
74 (uintptr_t)copyin_fast + setidt_disp);
75 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
76 (uintptr_t)copyout_fast + setidt_disp);
77 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
78 (uintptr_t)fubyte_fast + setidt_disp);
79 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
80 (uintptr_t)fuword16_fast + setidt_disp);
81 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
82 (uintptr_t)fueword_fast + setidt_disp);
83 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
84 (uintptr_t)subyte_fast + setidt_disp);
85 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
86 (uintptr_t)suword16_fast + setidt_disp);
87 suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
88 (uintptr_t)suword_fast + setidt_disp);
89 }
90
91 int
cp_slow0(vm_offset_t uva,size_t len,bool write,void (* f)(vm_offset_t,void *),void * arg)92 cp_slow0(vm_offset_t uva, size_t len, bool write,
93 void (*f)(vm_offset_t, void *), void *arg)
94 {
95 struct pcpu *pc;
96 vm_page_t m[2];
97 vm_offset_t kaddr;
98 int error, i, plen;
99 bool sleepable;
100
101 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
102 MPASS(plen <= nitems(m));
103 error = 0;
104 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
105 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
106 m, nitems(m));
107 if (i != plen)
108 return (EFAULT);
109 sched_pin();
110 pc = get_pcpu();
111 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
112 (curthread->td_pflags & TDP_NOFAULTING) != 0) {
113 sleepable = false;
114 mtx_lock(&pc->pc_copyout_mlock);
115 kaddr = pc->pc_copyout_maddr;
116 } else {
117 sleepable = true;
118 sx_xlock(&pc->pc_copyout_slock);
119 kaddr = pc->pc_copyout_saddr;
120 }
121 pmap_cp_slow0_map(kaddr, plen, m);
122 kaddr += uva - trunc_page(uva);
123 f(kaddr, arg);
124 sched_unpin();
125 if (sleepable)
126 sx_xunlock(&pc->pc_copyout_slock);
127 else
128 mtx_unlock(&pc->pc_copyout_mlock);
129 vm_page_unhold_pages(m, plen);
130 return (error);
131 }
132
133 struct copyinstr_arg0 {
134 vm_offset_t kc;
135 size_t len;
136 size_t alen;
137 bool end;
138 };
139
140 static void
copyinstr_slow0(vm_offset_t kva,void * arg)141 copyinstr_slow0(vm_offset_t kva, void *arg)
142 {
143 struct copyinstr_arg0 *ca;
144 char c;
145
146 ca = arg;
147 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
148 while (ca->alen < ca->len && !ca->end) {
149 c = *(char *)(kva + ca->alen);
150 *(char *)ca->kc = c;
151 ca->alen++;
152 ca->kc++;
153 if (c == '\0')
154 ca->end = true;
155 }
156 }
157
158 int
copyinstr(const void * udaddr,void * kaddr,size_t maxlen,size_t * lencopied)159 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
160 {
161 struct copyinstr_arg0 ca;
162 vm_offset_t uc;
163 size_t plen;
164 int error;
165
166 error = 0;
167 ca.end = false;
168 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
169 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
170 ca.len = round_page(uc) - uc;
171 if (ca.len == 0)
172 ca.len = PAGE_SIZE;
173 if (plen + ca.len > maxlen)
174 ca.len = maxlen - plen;
175 ca.alen = 0;
176 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
177 error = EFAULT;
178 break;
179 }
180 }
181 if (!ca.end && plen == maxlen && error == 0)
182 error = ENAMETOOLONG;
183 if (lencopied != NULL)
184 *lencopied = plen;
185 return (error);
186 }
187
188 struct copyin_arg0 {
189 vm_offset_t kc;
190 size_t len;
191 };
192
193 static void
copyin_slow0(vm_offset_t kva,void * arg)194 copyin_slow0(vm_offset_t kva, void *arg)
195 {
196 struct copyin_arg0 *ca;
197
198 ca = arg;
199 bcopy((void *)kva, (void *)ca->kc, ca->len);
200 }
201
202 int
copyin(const void * udaddr,void * kaddr,size_t len)203 copyin(const void *udaddr, void *kaddr, size_t len)
204 {
205 struct copyin_arg0 ca;
206 vm_offset_t uc;
207 size_t plen;
208
209 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
210 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
211 return (EFAULT);
212 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
213 copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0))
214 return (0);
215 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
216 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
217 ca.len = round_page(uc) - uc;
218 if (ca.len == 0)
219 ca.len = PAGE_SIZE;
220 if (plen + ca.len > len)
221 ca.len = len - plen;
222 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
223 return (EFAULT);
224 }
225 return (0);
226 }
227
228 static void
copyout_slow0(vm_offset_t kva,void * arg)229 copyout_slow0(vm_offset_t kva, void *arg)
230 {
231 struct copyin_arg0 *ca;
232
233 ca = arg;
234 bcopy((void *)ca->kc, (void *)kva, ca->len);
235 }
236
237 int
copyout(const void * kaddr,void * udaddr,size_t len)238 copyout(const void *kaddr, void *udaddr, size_t len)
239 {
240 struct copyin_arg0 ca;
241 vm_offset_t uc;
242 size_t plen;
243
244 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
245 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
246 return (EFAULT);
247 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
248 copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0))
249 return (0);
250 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr;
251 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
252 ca.len = round_page(uc) - uc;
253 if (ca.len == 0)
254 ca.len = PAGE_SIZE;
255 if (plen + ca.len > len)
256 ca.len = len - plen;
257 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
258 return (EFAULT);
259 }
260 return (0);
261 }
262
263 /*
264 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
265 * memory.
266 */
267
268 static void
fubyte_slow0(vm_offset_t kva,void * arg)269 fubyte_slow0(vm_offset_t kva, void *arg)
270 {
271
272 *(int *)arg = *(u_char *)kva;
273 }
274
275 int
fubyte(volatile const void * base)276 fubyte(volatile const void *base)
277 {
278 int res;
279
280 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
281 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
282 return (-1);
283 if (fast_copyout) {
284 res = fubyte_fast_tramp(base, pmap_get_kcr3());
285 if (res != -1)
286 return (res);
287 }
288 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
289 &res) != 0)
290 return (-1);
291 return (res);
292 }
293
294 static void
fuword16_slow0(vm_offset_t kva,void * arg)295 fuword16_slow0(vm_offset_t kva, void *arg)
296 {
297
298 *(int *)arg = *(uint16_t *)kva;
299 }
300
301 int
fuword16(volatile const void * base)302 fuword16(volatile const void *base)
303 {
304 int res;
305
306 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
307 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
308 return (-1);
309 if (fast_copyout) {
310 res = fuword16_fast_tramp(base, pmap_get_kcr3());
311 if (res != -1)
312 return (res);
313 }
314 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
315 fuword16_slow0, &res) != 0)
316 return (-1);
317 return (res);
318 }
319
320 static void
fueword_slow0(vm_offset_t kva,void * arg)321 fueword_slow0(vm_offset_t kva, void *arg)
322 {
323
324 *(uint32_t *)arg = *(uint32_t *)kva;
325 }
326
327 int
fueword(volatile const void * base,long * val)328 fueword(volatile const void *base, long *val)
329 {
330 uint32_t res;
331
332 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
333 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
334 return (-1);
335 if (fast_copyout) {
336 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0)
337 return (0);
338 }
339 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
340 &res) != 0)
341 return (-1);
342 *val = res;
343 return (0);
344 }
345
346 int
fueword32(volatile const void * base,int32_t * val)347 fueword32(volatile const void *base, int32_t *val)
348 {
349
350 return (fueword(base, (long *)val));
351 }
352
353 /*
354 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
355 */
356
357 static void
subyte_slow0(vm_offset_t kva,void * arg)358 subyte_slow0(vm_offset_t kva, void *arg)
359 {
360
361 *(u_char *)kva = *(int *)arg;
362 }
363
364 int
subyte(volatile void * base,int byte)365 subyte(volatile void *base, int byte)
366 {
367
368 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
369 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
370 return (-1);
371 if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0)
372 return (0);
373 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
374 &byte) != 0 ? -1 : 0);
375 }
376
377 static void
suword16_slow0(vm_offset_t kva,void * arg)378 suword16_slow0(vm_offset_t kva, void *arg)
379 {
380
381 *(int *)kva = *(uint16_t *)arg;
382 }
383
384 int
suword16(volatile void * base,int word)385 suword16(volatile void *base, int word)
386 {
387
388 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
389 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
390 return (-1);
391 if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3())
392 == 0)
393 return (0);
394 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
395 suword16_slow0, &word) != 0 ? -1 : 0);
396 }
397
398 static void
suword_slow0(vm_offset_t kva,void * arg)399 suword_slow0(vm_offset_t kva, void *arg)
400 {
401
402 *(int *)kva = *(uint32_t *)arg;
403 }
404
405 int
suword(volatile void * base,long word)406 suword(volatile void *base, long word)
407 {
408
409 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
410 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
411 return (-1);
412 if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0)
413 return (0);
414 return (cp_slow0((vm_offset_t)base, sizeof(long), true,
415 suword_slow0, &word) != 0 ? -1 : 0);
416 }
417
418 int
suword32(volatile void * base,int32_t word)419 suword32(volatile void *base, int32_t word)
420 {
421
422 return (suword(base, word));
423 }
424
425 struct casueword_arg0 {
426 uint32_t oldval;
427 uint32_t newval;
428 int res;
429 };
430
431 static void
casueword_slow0(vm_offset_t kva,void * arg)432 casueword_slow0(vm_offset_t kva, void *arg)
433 {
434 struct casueword_arg0 *ca;
435
436 ca = arg;
437 ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
438 ca->newval);
439 }
440
441 int
casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)442 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
443 uint32_t newval)
444 {
445 struct casueword_arg0 ca;
446 int res;
447
448 ca.oldval = oldval;
449 ca.newval = newval;
450 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
451 casueword_slow0, &ca);
452 if (res == 0) {
453 *oldvalp = ca.oldval;
454 return (ca.res);
455 }
456 return (-1);
457 }
458
459 int
casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)460 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
461 {
462 struct casueword_arg0 ca;
463 int res;
464
465 ca.oldval = oldval;
466 ca.newval = newval;
467 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
468 casueword_slow0, &ca);
469 if (res == 0) {
470 *oldvalp = ca.oldval;
471 return (ca.res);
472 }
473 return (-1);
474 }
475