1 /* $OpenBSD: criov.c,v 1.9 2002/01/29 15:48:29 jason Exp $ */
2
3 /*-
4 * Copyright (c) 1999 Theo de Raadt
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/proc.h>
34 #include <sys/errno.h>
35 #include <sys/malloc.h>
36 #include <sys/kernel.h>
37 #include <sys/mbuf.h>
38 #include <sys/uio.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/sdt.h>
42
43 #include <machine/vmparam.h>
44
45 #include <vm/vm.h>
46 #include <vm/vm_page.h>
47 #include <vm/pmap.h>
48
49 #include <opencrypto/cryptodev.h>
50
51 SDT_PROVIDER_DECLARE(opencrypto);
52
53 /*
54 * These macros are only for avoiding code duplication, as we need to skip
55 * given number of bytes in the same way in several functions below.
56 */
57 #define CUIO_SKIP() do { \
58 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
59 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
60 while (off > 0) { \
61 KASSERT(iol >= 0, ("%s: empty in skip", __func__)); \
62 if (off < iov->iov_len) \
63 break; \
64 off -= iov->iov_len; \
65 iol--; \
66 iov++; \
67 } \
68 } while (0)
69
70 #define CVM_PAGE_SKIP() do { \
71 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
72 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
73 while (off > 0) { \
74 if (off < PAGE_SIZE) \
75 break; \
76 processed += PAGE_SIZE - off; \
77 off -= PAGE_SIZE - off; \
78 pages++; \
79 } \
80 } while (0)
81
82 static void
cuio_copydata(struct uio * uio,int off,int len,caddr_t cp)83 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
84 {
85 struct iovec *iov = uio->uio_iov;
86 int iol __diagused = uio->uio_iovcnt;
87 unsigned count;
88
89 CUIO_SKIP();
90 while (len > 0) {
91 KASSERT(iol >= 0, ("%s: empty", __func__));
92 count = min(iov->iov_len - off, len);
93 bcopy(((caddr_t)iov->iov_base) + off, cp, count);
94 len -= count;
95 cp += count;
96 off = 0;
97 iol--;
98 iov++;
99 }
100 }
101
102 static void
cuio_copyback(struct uio * uio,int off,int len,c_caddr_t cp)103 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
104 {
105 struct iovec *iov = uio->uio_iov;
106 int iol __diagused = uio->uio_iovcnt;
107 unsigned count;
108
109 CUIO_SKIP();
110 while (len > 0) {
111 KASSERT(iol >= 0, ("%s: empty", __func__));
112 count = min(iov->iov_len - off, len);
113 bcopy(cp, ((caddr_t)iov->iov_base) + off, count);
114 len -= count;
115 cp += count;
116 off = 0;
117 iol--;
118 iov++;
119 }
120 }
121
122 /*
123 * Return the index and offset of location in iovec list.
124 */
125 static int
cuio_getptr(struct uio * uio,int loc,int * off)126 cuio_getptr(struct uio *uio, int loc, int *off)
127 {
128 int ind, len;
129
130 ind = 0;
131 while (loc >= 0 && ind < uio->uio_iovcnt) {
132 len = uio->uio_iov[ind].iov_len;
133 if (len > loc) {
134 *off = loc;
135 return (ind);
136 }
137 loc -= len;
138 ind++;
139 }
140
141 if (ind > 0 && loc == 0) {
142 ind--;
143 *off = uio->uio_iov[ind].iov_len;
144 return (ind);
145 }
146
147 return (-1);
148 }
149
150 #if CRYPTO_MAY_HAVE_VMPAGE
151 /*
152 * Apply function f to the data in a vm_page_t list starting "off" bytes from
153 * the beginning, continuing for "len" bytes.
154 */
155 static int
cvm_page_apply(vm_page_t * pages,int off,int len,int (* f)(void *,const void *,u_int),void * arg)156 cvm_page_apply(vm_page_t *pages, int off, int len,
157 int (*f)(void *, const void *, u_int), void *arg)
158 {
159 int processed = 0;
160 unsigned count;
161 int rval;
162
163 CVM_PAGE_SKIP();
164 while (len > 0) {
165 char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
166 count = min(PAGE_SIZE - off, len);
167 rval = (*f)(arg, kaddr + off, count);
168 if (rval)
169 return (rval);
170 len -= count;
171 processed += count;
172 off = 0;
173 pages++;
174 }
175 return (0);
176 }
177
178 static inline void *
cvm_page_contiguous_segment(vm_page_t * pages,size_t skip,int len)179 cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
180 {
181 if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
182 return (NULL);
183
184 pages += (skip / PAGE_SIZE);
185 skip -= rounddown(skip, PAGE_SIZE);
186 return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
187 }
188
189 /*
190 * Copy len bytes of data from the vm_page_t array, skipping the first off
191 * bytes, into the pointer cp. Return the number of bytes skipped and copied.
192 * Does not verify the length of the array.
193 */
194 static int
cvm_page_copyback(vm_page_t * pages,int off,int len,c_caddr_t cp)195 cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
196 {
197 int processed = 0;
198 unsigned count;
199
200 CVM_PAGE_SKIP();
201 while (len > 0) {
202 count = min(PAGE_SIZE - off, len);
203 bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
204 count);
205 len -= count;
206 cp += count;
207 processed += count;
208 off = 0;
209 pages++;
210 }
211 return (processed);
212 }
213
214 /*
215 * Copy len bytes of data from the pointer cp into the vm_page_t array,
216 * skipping the first off bytes, Return the number of bytes skipped and copied.
217 * Does not verify the length of the array.
218 */
219 static int
cvm_page_copydata(vm_page_t * pages,int off,int len,caddr_t cp)220 cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
221 {
222 int processed = 0;
223 unsigned count;
224
225 CVM_PAGE_SKIP();
226 while (len > 0) {
227 count = min(PAGE_SIZE - off, len);
228 bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
229 count);
230 len -= count;
231 cp += count;
232 processed += count;
233 off = 0;
234 pages++;
235 }
236 return processed;
237 }
238 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
239
240 /*
241 * Given a starting page in an m_epg, determine the length of the
242 * current physically contiguous segment.
243 */
244 static __inline size_t
m_epg_pages_extent(struct mbuf * m,int idx,u_int pglen)245 m_epg_pages_extent(struct mbuf *m, int idx, u_int pglen)
246 {
247 size_t len;
248 u_int i;
249
250 len = pglen;
251 for (i = idx + 1; i < m->m_epg_npgs; i++) {
252 if (m->m_epg_pa[i - 1] + PAGE_SIZE != m->m_epg_pa[i])
253 break;
254 len += m_epg_pagelen(m, i, 0);
255 }
256 return (len);
257 }
258
259 static void *
m_epg_segment(struct mbuf * m,size_t offset,size_t * len)260 m_epg_segment(struct mbuf *m, size_t offset, size_t *len)
261 {
262 u_int i, pglen, pgoff;
263
264 offset += mtod(m, vm_offset_t);
265 if (offset < m->m_epg_hdrlen) {
266 *len = m->m_epg_hdrlen - offset;
267 return (m->m_epg_hdr + offset);
268 }
269 offset -= m->m_epg_hdrlen;
270 pgoff = m->m_epg_1st_off;
271 for (i = 0; i < m->m_epg_npgs; i++) {
272 pglen = m_epg_pagelen(m, i, pgoff);
273 if (offset < pglen) {
274 *len = m_epg_pages_extent(m, i, pglen) - offset;
275 return ((void *)PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff +
276 offset));
277 }
278 offset -= pglen;
279 pgoff = 0;
280 }
281 KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer",
282 __func__));
283 *len = m->m_epg_trllen - offset;
284 return (m->m_epg_trail + offset);
285 }
286
287 static __inline void *
m_epg_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)288 m_epg_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
289 {
290 void *base;
291 size_t seglen;
292
293 base = m_epg_segment(m, skip, &seglen);
294 if (len > seglen)
295 return (NULL);
296 return (base);
297 }
298
299 void
crypto_cursor_init(struct crypto_buffer_cursor * cc,const struct crypto_buffer * cb)300 crypto_cursor_init(struct crypto_buffer_cursor *cc,
301 const struct crypto_buffer *cb)
302 {
303 memset(cc, 0, sizeof(*cc));
304 cc->cc_type = cb->cb_type;
305 switch (cc->cc_type) {
306 case CRYPTO_BUF_CONTIG:
307 cc->cc_buf = cb->cb_buf;
308 cc->cc_buf_len = cb->cb_buf_len;
309 break;
310 case CRYPTO_BUF_MBUF:
311 case CRYPTO_BUF_SINGLE_MBUF:
312 cc->cc_mbuf = cb->cb_mbuf;
313 break;
314 case CRYPTO_BUF_VMPAGE:
315 cc->cc_vmpage = cb->cb_vm_page;
316 cc->cc_buf_len = cb->cb_vm_page_len;
317 cc->cc_offset = cb->cb_vm_page_offset;
318 break;
319 case CRYPTO_BUF_UIO:
320 cc->cc_iov = cb->cb_uio->uio_iov;
321 cc->cc_buf_len = cb->cb_uio->uio_resid;
322 break;
323 default:
324 #ifdef INVARIANTS
325 panic("%s: invalid buffer type %d", __func__, cb->cb_type);
326 #endif
327 break;
328 }
329 }
330
331 SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
332
333 void
crypto_cursor_advance(struct crypto_buffer_cursor * cc,size_t amount)334 crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
335 {
336 size_t remain;
337
338 switch (cc->cc_type) {
339 case CRYPTO_BUF_CONTIG:
340 MPASS(cc->cc_buf_len >= amount);
341 cc->cc_buf += amount;
342 cc->cc_buf_len -= amount;
343 break;
344 case CRYPTO_BUF_MBUF:
345 for (;;) {
346 remain = cc->cc_mbuf->m_len - cc->cc_offset;
347 if (amount < remain) {
348 cc->cc_offset += amount;
349 break;
350 }
351 amount -= remain;
352 cc->cc_mbuf = cc->cc_mbuf->m_next;
353 cc->cc_offset = 0;
354 if (amount == 0)
355 break;
356 }
357 break;
358 case CRYPTO_BUF_SINGLE_MBUF:
359 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
360 cc->cc_offset += amount;
361 break;
362 case CRYPTO_BUF_VMPAGE:
363 for (;;) {
364 SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
365 cc, amount);
366 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
367 if (amount < remain) {
368 cc->cc_buf_len -= amount;
369 cc->cc_offset += amount;
370 break;
371 }
372 cc->cc_buf_len -= remain;
373 amount -= remain;
374 cc->cc_vmpage++;
375 cc->cc_offset = 0;
376 if (amount == 0 || cc->cc_buf_len == 0)
377 break;
378 }
379 break;
380 case CRYPTO_BUF_UIO:
381 for (;;) {
382 remain = cc->cc_iov->iov_len - cc->cc_offset;
383 if (amount < remain) {
384 cc->cc_offset += amount;
385 break;
386 }
387 cc->cc_buf_len -= remain;
388 amount -= remain;
389 cc->cc_iov++;
390 cc->cc_offset = 0;
391 if (amount == 0)
392 break;
393 }
394 break;
395 default:
396 #ifdef INVARIANTS
397 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
398 #endif
399 break;
400 }
401 }
402
403 void *
crypto_cursor_segment(struct crypto_buffer_cursor * cc,size_t * len)404 crypto_cursor_segment(struct crypto_buffer_cursor *cc, size_t *len)
405 {
406 switch (cc->cc_type) {
407 case CRYPTO_BUF_CONTIG:
408 case CRYPTO_BUF_UIO:
409 case CRYPTO_BUF_VMPAGE:
410 if (cc->cc_buf_len == 0) {
411 *len = 0;
412 return (NULL);
413 }
414 break;
415 case CRYPTO_BUF_MBUF:
416 case CRYPTO_BUF_SINGLE_MBUF:
417 if (cc->cc_mbuf == NULL) {
418 *len = 0;
419 return (NULL);
420 }
421 break;
422 default:
423 #ifdef INVARIANTS
424 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
425 #endif
426 *len = 0;
427 return (NULL);
428 }
429
430 switch (cc->cc_type) {
431 case CRYPTO_BUF_CONTIG:
432 *len = cc->cc_buf_len;
433 return (cc->cc_buf);
434 case CRYPTO_BUF_MBUF:
435 case CRYPTO_BUF_SINGLE_MBUF:
436 if (cc->cc_mbuf->m_flags & M_EXTPG)
437 return (m_epg_segment(cc->cc_mbuf, cc->cc_offset, len));
438 *len = cc->cc_mbuf->m_len - cc->cc_offset;
439 return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
440 case CRYPTO_BUF_VMPAGE:
441 *len = PAGE_SIZE - cc->cc_offset;
442 return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
443 *cc->cc_vmpage)) + cc->cc_offset);
444 case CRYPTO_BUF_UIO:
445 *len = cc->cc_iov->iov_len - cc->cc_offset;
446 return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
447 default:
448 __assert_unreachable();
449 }
450 }
451
452 void *
crypto_cursor_segbase(struct crypto_buffer_cursor * cc)453 crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
454 {
455 size_t len;
456
457 return (crypto_cursor_segment(cc, &len));
458 }
459
460 size_t
crypto_cursor_seglen(struct crypto_buffer_cursor * cc)461 crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
462 {
463 size_t len;
464
465 crypto_cursor_segment(cc, &len);
466 return (len);
467 }
468
469 void
crypto_cursor_copyback(struct crypto_buffer_cursor * cc,int size,const void * vsrc)470 crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
471 const void *vsrc)
472 {
473 size_t remain, todo;
474 const char *src;
475 char *dst;
476
477 src = vsrc;
478 switch (cc->cc_type) {
479 case CRYPTO_BUF_CONTIG:
480 MPASS(cc->cc_buf_len >= size);
481 memcpy(cc->cc_buf, src, size);
482 cc->cc_buf += size;
483 cc->cc_buf_len -= size;
484 break;
485 case CRYPTO_BUF_MBUF:
486 for (;;) {
487 /*
488 * This uses m_copyback() for individual
489 * mbufs so that cc_mbuf and cc_offset are
490 * updated.
491 */
492 remain = cc->cc_mbuf->m_len - cc->cc_offset;
493 todo = MIN(remain, size);
494 m_copyback(cc->cc_mbuf, cc->cc_offset, todo, src);
495 src += todo;
496 if (todo < remain) {
497 cc->cc_offset += todo;
498 break;
499 }
500 size -= todo;
501 cc->cc_mbuf = cc->cc_mbuf->m_next;
502 cc->cc_offset = 0;
503 if (size == 0)
504 break;
505 }
506 break;
507 case CRYPTO_BUF_SINGLE_MBUF:
508 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
509 m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
510 cc->cc_offset += size;
511 break;
512 case CRYPTO_BUF_VMPAGE:
513 for (;;) {
514 dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
515 *cc->cc_vmpage)) + cc->cc_offset;
516 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
517 todo = MIN(remain, size);
518 memcpy(dst, src, todo);
519 src += todo;
520 cc->cc_buf_len -= todo;
521 if (todo < remain) {
522 cc->cc_offset += todo;
523 break;
524 }
525 size -= todo;
526 cc->cc_vmpage++;
527 cc->cc_offset = 0;
528 if (size == 0)
529 break;
530 }
531 break;
532 case CRYPTO_BUF_UIO:
533 for (;;) {
534 dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
535 remain = cc->cc_iov->iov_len - cc->cc_offset;
536 todo = MIN(remain, size);
537 memcpy(dst, src, todo);
538 src += todo;
539 cc->cc_buf_len -= todo;
540 if (todo < remain) {
541 cc->cc_offset += todo;
542 break;
543 }
544 size -= todo;
545 cc->cc_iov++;
546 cc->cc_offset = 0;
547 if (size == 0)
548 break;
549 }
550 break;
551 default:
552 #ifdef INVARIANTS
553 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
554 #endif
555 break;
556 }
557 }
558
559 void
crypto_cursor_copydata(struct crypto_buffer_cursor * cc,int size,void * vdst)560 crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
561 {
562 size_t remain, todo;
563 const char *src;
564 char *dst;
565
566 dst = vdst;
567 switch (cc->cc_type) {
568 case CRYPTO_BUF_CONTIG:
569 MPASS(cc->cc_buf_len >= size);
570 memcpy(dst, cc->cc_buf, size);
571 cc->cc_buf += size;
572 cc->cc_buf_len -= size;
573 break;
574 case CRYPTO_BUF_MBUF:
575 for (;;) {
576 /*
577 * This uses m_copydata() for individual
578 * mbufs so that cc_mbuf and cc_offset are
579 * updated.
580 */
581 remain = cc->cc_mbuf->m_len - cc->cc_offset;
582 todo = MIN(remain, size);
583 m_copydata(cc->cc_mbuf, cc->cc_offset, todo, dst);
584 dst += todo;
585 if (todo < remain) {
586 cc->cc_offset += todo;
587 break;
588 }
589 size -= todo;
590 cc->cc_mbuf = cc->cc_mbuf->m_next;
591 cc->cc_offset = 0;
592 if (size == 0)
593 break;
594 }
595 break;
596 case CRYPTO_BUF_SINGLE_MBUF:
597 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
598 m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
599 cc->cc_offset += size;
600 break;
601 case CRYPTO_BUF_VMPAGE:
602 for (;;) {
603 src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
604 *cc->cc_vmpage)) + cc->cc_offset;
605 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
606 todo = MIN(remain, size);
607 memcpy(dst, src, todo);
608 dst += todo;
609 cc->cc_buf_len -= todo;
610 if (todo < remain) {
611 cc->cc_offset += todo;
612 break;
613 }
614 size -= todo;
615 cc->cc_vmpage++;
616 cc->cc_offset = 0;
617 if (size == 0)
618 break;
619 }
620 break;
621 case CRYPTO_BUF_UIO:
622 for (;;) {
623 src = (const char *)cc->cc_iov->iov_base +
624 cc->cc_offset;
625 remain = cc->cc_iov->iov_len - cc->cc_offset;
626 todo = MIN(remain, size);
627 memcpy(dst, src, todo);
628 dst += todo;
629 cc->cc_buf_len -= todo;
630 if (todo < remain) {
631 cc->cc_offset += todo;
632 break;
633 }
634 size -= todo;
635 cc->cc_iov++;
636 cc->cc_offset = 0;
637 if (size == 0)
638 break;
639 }
640 break;
641 default:
642 #ifdef INVARIANTS
643 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
644 #endif
645 break;
646 }
647 }
648
649 /*
650 * To avoid advancing 'cursor', make a local copy that gets advanced
651 * instead.
652 */
653 void
crypto_cursor_copydata_noadv(struct crypto_buffer_cursor * cc,int size,void * vdst)654 crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
655 void *vdst)
656 {
657 struct crypto_buffer_cursor copy;
658
659 copy = *cc;
660 crypto_cursor_copydata(©, size, vdst);
661 }
662
663 /*
664 * Apply function f to the data in an iovec list starting "off" bytes from
665 * the beginning, continuing for "len" bytes.
666 */
667 static int
cuio_apply(struct uio * uio,int off,int len,int (* f)(void *,const void *,u_int),void * arg)668 cuio_apply(struct uio *uio, int off, int len,
669 int (*f)(void *, const void *, u_int), void *arg)
670 {
671 struct iovec *iov = uio->uio_iov;
672 int iol __diagused = uio->uio_iovcnt;
673 unsigned count;
674 int rval;
675
676 CUIO_SKIP();
677 while (len > 0) {
678 KASSERT(iol >= 0, ("%s: empty", __func__));
679 count = min(iov->iov_len - off, len);
680 rval = (*f)(arg, ((caddr_t)iov->iov_base) + off, count);
681 if (rval)
682 return (rval);
683 len -= count;
684 off = 0;
685 iol--;
686 iov++;
687 }
688 return (0);
689 }
690
691 void
crypto_copyback(struct cryptop * crp,int off,int size,const void * src)692 crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
693 {
694 struct crypto_buffer *cb;
695
696 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE)
697 cb = &crp->crp_obuf;
698 else
699 cb = &crp->crp_buf;
700 switch (cb->cb_type) {
701 case CRYPTO_BUF_MBUF:
702 case CRYPTO_BUF_SINGLE_MBUF:
703 m_copyback(cb->cb_mbuf, off, size, src);
704 break;
705 #if CRYPTO_MAY_HAVE_VMPAGE
706 case CRYPTO_BUF_VMPAGE:
707 MPASS(size <= cb->cb_vm_page_len);
708 MPASS(size + off <=
709 cb->cb_vm_page_len + cb->cb_vm_page_offset);
710 cvm_page_copyback(cb->cb_vm_page,
711 off + cb->cb_vm_page_offset, size, src);
712 break;
713 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
714 case CRYPTO_BUF_UIO:
715 cuio_copyback(cb->cb_uio, off, size, src);
716 break;
717 case CRYPTO_BUF_CONTIG:
718 MPASS(off + size <= cb->cb_buf_len);
719 bcopy(src, cb->cb_buf + off, size);
720 break;
721 default:
722 #ifdef INVARIANTS
723 panic("invalid crp buf type %d", cb->cb_type);
724 #endif
725 break;
726 }
727 }
728
729 void
crypto_copydata(struct cryptop * crp,int off,int size,void * dst)730 crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
731 {
732
733 switch (crp->crp_buf.cb_type) {
734 case CRYPTO_BUF_MBUF:
735 case CRYPTO_BUF_SINGLE_MBUF:
736 m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
737 break;
738 #if CRYPTO_MAY_HAVE_VMPAGE
739 case CRYPTO_BUF_VMPAGE:
740 MPASS(size <= crp->crp_buf.cb_vm_page_len);
741 MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
742 crp->crp_buf.cb_vm_page_offset);
743 cvm_page_copydata(crp->crp_buf.cb_vm_page,
744 off + crp->crp_buf.cb_vm_page_offset, size, dst);
745 break;
746 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
747 case CRYPTO_BUF_UIO:
748 cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
749 break;
750 case CRYPTO_BUF_CONTIG:
751 MPASS(off + size <= crp->crp_buf.cb_buf_len);
752 bcopy(crp->crp_buf.cb_buf + off, dst, size);
753 break;
754 default:
755 #ifdef INVARIANTS
756 panic("invalid crp buf type %d", crp->crp_buf.cb_type);
757 #endif
758 break;
759 }
760 }
761
762 int
crypto_apply_buf(struct crypto_buffer * cb,int off,int len,int (* f)(void *,const void *,u_int),void * arg)763 crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
764 int (*f)(void *, const void *, u_int), void *arg)
765 {
766 int error;
767
768 switch (cb->cb_type) {
769 case CRYPTO_BUF_MBUF:
770 case CRYPTO_BUF_SINGLE_MBUF:
771 error = m_apply(cb->cb_mbuf, off, len,
772 (int (*)(void *, void *, u_int))f, arg);
773 break;
774 case CRYPTO_BUF_UIO:
775 error = cuio_apply(cb->cb_uio, off, len, f, arg);
776 break;
777 #if CRYPTO_MAY_HAVE_VMPAGE
778 case CRYPTO_BUF_VMPAGE:
779 error = cvm_page_apply(cb->cb_vm_page,
780 off + cb->cb_vm_page_offset, len, f, arg);
781 break;
782 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
783 case CRYPTO_BUF_CONTIG:
784 MPASS(off + len <= cb->cb_buf_len);
785 error = (*f)(arg, cb->cb_buf + off, len);
786 break;
787 default:
788 #ifdef INVARIANTS
789 panic("invalid crypto buf type %d", cb->cb_type);
790 #endif
791 error = 0;
792 break;
793 }
794 return (error);
795 }
796
797 int
crypto_apply(struct cryptop * crp,int off,int len,int (* f)(void *,const void *,u_int),void * arg)798 crypto_apply(struct cryptop *crp, int off, int len,
799 int (*f)(void *, const void *, u_int), void *arg)
800 {
801 return (crypto_apply_buf(&crp->crp_buf, off, len, f, arg));
802 }
803
804 static inline void *
m_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)805 m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
806 {
807 int rel_off;
808
809 MPASS(skip <= INT_MAX);
810
811 m = m_getptr(m, (int)skip, &rel_off);
812 if (m == NULL)
813 return (NULL);
814
815 MPASS(rel_off >= 0);
816 skip = rel_off;
817 if (skip + len > m->m_len)
818 return (NULL);
819
820 if (m->m_flags & M_EXTPG)
821 return (m_epg_contiguous_subsegment(m, skip, len));
822 return (mtod(m, char*) + skip);
823 }
824
825 static inline void *
cuio_contiguous_segment(struct uio * uio,size_t skip,size_t len)826 cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
827 {
828 int rel_off, idx;
829
830 MPASS(skip <= INT_MAX);
831 idx = cuio_getptr(uio, (int)skip, &rel_off);
832 if (idx < 0)
833 return (NULL);
834
835 MPASS(rel_off >= 0);
836 skip = rel_off;
837 if (skip + len > uio->uio_iov[idx].iov_len)
838 return (NULL);
839 return ((char *)uio->uio_iov[idx].iov_base + skip);
840 }
841
842 void *
crypto_buffer_contiguous_subsegment(struct crypto_buffer * cb,size_t skip,size_t len)843 crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
844 size_t len)
845 {
846
847 switch (cb->cb_type) {
848 case CRYPTO_BUF_MBUF:
849 case CRYPTO_BUF_SINGLE_MBUF:
850 return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
851 case CRYPTO_BUF_UIO:
852 return (cuio_contiguous_segment(cb->cb_uio, skip, len));
853 #if CRYPTO_MAY_HAVE_VMPAGE
854 case CRYPTO_BUF_VMPAGE:
855 MPASS(skip + len <= cb->cb_vm_page_len);
856 return (cvm_page_contiguous_segment(cb->cb_vm_page,
857 skip + cb->cb_vm_page_offset, len));
858 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
859 case CRYPTO_BUF_CONTIG:
860 MPASS(skip + len <= cb->cb_buf_len);
861 return (cb->cb_buf + skip);
862 default:
863 #ifdef INVARIANTS
864 panic("invalid crp buf type %d", cb->cb_type);
865 #endif
866 return (NULL);
867 }
868 }
869
870 void *
crypto_contiguous_subsegment(struct cryptop * crp,size_t skip,size_t len)871 crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
872 {
873 return (crypto_buffer_contiguous_subsegment(&crp->crp_buf, skip, len));
874 }
875