1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */
2 /*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/stdint.h>
28 #include <sys/param.h>
29 #include <sys/queue.h>
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/bus.h>
34 #include <sys/module.h>
35 #include <sys/lock.h>
36 #include <sys/condvar.h>
37 #include <sys/sysctl.h>
38 #include <sys/unistd.h>
39 #include <sys/callout.h>
40 #include <sys/malloc.h>
41 #include <sys/caps.h>
42
43 #include <bus/u4b/usb.h>
44 #include <bus/u4b/usbdi.h>
45 #include <bus/u4b/usbdi_util.h>
46
47 #define USB_DEBUG_VAR usb_debug
48
49 #include <bus/u4b/usb_core.h>
50 #include <bus/u4b/usb_busdma.h>
51 #include <bus/u4b/usb_process.h>
52 #include <bus/u4b/usb_transfer.h>
53 #include <bus/u4b/usb_device.h>
54 #include <bus/u4b/usb_util.h>
55 #include <bus/u4b/usb_debug.h>
56
57 #include <bus/u4b/usb_controller.h>
58 #include <bus/u4b/usb_bus.h>
59
60 #if USB_HAVE_BUSDMA
61 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
62 static void usb_dma_tag_destroy(struct usb_dma_tag *);
63 #if 0
64 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t);
65 #endif
66 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
67 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
68 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
69 uint8_t);
70 #endif
71
72 /*------------------------------------------------------------------------*
73 * usbd_get_page - lookup DMA-able memory for the given offset
74 *
75 * NOTE: Only call this function when the "page_cache" structure has
76 * been properly initialized !
77 *------------------------------------------------------------------------*/
78 void
usbd_get_page(struct usb_page_cache * pc,usb_frlength_t offset,struct usb_page_search * res)79 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
80 struct usb_page_search *res)
81 {
82 #if USB_HAVE_BUSDMA
83 struct usb_page *page;
84
85 if (pc->page_start) {
86
87 /* Case 1 - something has been loaded into DMA */
88
89 if (pc->buffer) {
90
91 /* Case 1a - Kernel Virtual Address */
92
93 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
94 }
95 offset += pc->page_offset_buf;
96
97 /* compute destination page */
98
99 page = pc->page_start;
100
101 if (pc->ismultiseg) {
102
103 page += (offset / USB_PAGE_SIZE);
104
105 offset %= USB_PAGE_SIZE;
106
107 res->length = USB_PAGE_SIZE - offset;
108 res->physaddr = page->physaddr + offset;
109 } else {
110 res->length = (usb_size_t)-1;
111 res->physaddr = page->physaddr + offset;
112 }
113 if (!pc->buffer) {
114
115 /* Case 1b - Non Kernel Virtual Address */
116
117 res->buffer = USB_ADD_BYTES(page->buffer, offset);
118 }
119 return;
120 }
121 #endif
122 /* Case 2 - Plain PIO */
123
124 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
125 res->length = (usb_size_t)-1;
126 #if USB_HAVE_BUSDMA
127 res->physaddr = 0;
128 #endif
129 }
130
131 /*------------------------------------------------------------------------*
132 * usbd_copy_in - copy directly to DMA-able memory
133 *------------------------------------------------------------------------*/
134 void
usbd_copy_in(struct usb_page_cache * cache,usb_frlength_t offset,const void * ptr,usb_frlength_t len)135 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
136 const void *ptr, usb_frlength_t len)
137 {
138 struct usb_page_search buf_res;
139
140 while (len != 0) {
141
142 usbd_get_page(cache, offset, &buf_res);
143
144 if (buf_res.length > len) {
145 buf_res.length = len;
146 }
147 memcpy(buf_res.buffer, ptr, buf_res.length);
148
149 offset += buf_res.length;
150 len -= buf_res.length;
151 ptr = USB_ADD_BYTES(ptr, buf_res.length);
152 }
153 }
154
155 /*------------------------------------------------------------------------*
156 * usbd_copy_in_user - copy directly to DMA-able memory from userland
157 *
158 * Return values:
159 * 0: Success
160 * Else: Failure
161 *------------------------------------------------------------------------*/
162 #if USB_HAVE_USER_IO
163 int
usbd_copy_in_user(struct usb_page_cache * cache,usb_frlength_t offset,const void * ptr,usb_frlength_t len)164 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
165 const void *ptr, usb_frlength_t len)
166 {
167 struct usb_page_search buf_res;
168 int error;
169
170 while (len != 0) {
171
172 usbd_get_page(cache, offset, &buf_res);
173
174 if (buf_res.length > len) {
175 buf_res.length = len;
176 }
177 error = copyin(ptr, buf_res.buffer, buf_res.length);
178 if (error)
179 return (error);
180
181 offset += buf_res.length;
182 len -= buf_res.length;
183 ptr = USB_ADD_BYTES(ptr, buf_res.length);
184 }
185 return (0); /* success */
186 }
187 #endif
188
189 /*------------------------------------------------------------------------*
190 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
191 *------------------------------------------------------------------------*/
192 #if USB_HAVE_MBUF
193 struct usb_m_copy_in_arg {
194 struct usb_page_cache *cache;
195 usb_frlength_t dst_offset;
196 };
197
198 static int
usbd_m_copy_in_cb(void * arg,void * src,uint32_t count)199 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
200 {
201 register struct usb_m_copy_in_arg *ua = arg;
202
203 usbd_copy_in(ua->cache, ua->dst_offset, src, count);
204 ua->dst_offset += count;
205 return (0);
206 }
207
208 void
usbd_m_copy_in(struct usb_page_cache * cache,usb_frlength_t dst_offset,struct mbuf * m,usb_size_t src_offset,usb_frlength_t src_len)209 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
210 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
211 {
212 struct usb_m_copy_in_arg arg = {cache, dst_offset};
213 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
214 }
215 #endif
216
217 /*------------------------------------------------------------------------*
218 * usb_uiomove - factored out code
219 *------------------------------------------------------------------------*/
220 #if USB_HAVE_USER_IO
221 int
usb_uiomove(struct usb_page_cache * pc,struct uio * uio,usb_frlength_t pc_offset,usb_frlength_t len)222 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
223 usb_frlength_t pc_offset, usb_frlength_t len)
224 {
225 struct usb_page_search res;
226 int error = 0;
227
228 while (len != 0) {
229
230 usbd_get_page(pc, pc_offset, &res);
231
232 if (res.length > len) {
233 res.length = len;
234 }
235 /*
236 * "uiomove()" can sleep so one needs to make a wrapper,
237 * exiting the mutex and checking things
238 */
239 error = uiomove(res.buffer, res.length, uio);
240
241 if (error) {
242 break;
243 }
244 pc_offset += res.length;
245 len -= res.length;
246 }
247 return (error);
248 }
249 #endif
250
251 /*------------------------------------------------------------------------*
252 * usbd_copy_out - copy directly from DMA-able memory
253 *------------------------------------------------------------------------*/
254 void
usbd_copy_out(struct usb_page_cache * cache,usb_frlength_t offset,void * ptr,usb_frlength_t len)255 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
256 void *ptr, usb_frlength_t len)
257 {
258 struct usb_page_search res;
259
260 while (len != 0) {
261
262 usbd_get_page(cache, offset, &res);
263
264 if (res.length > len) {
265 res.length = len;
266 }
267 memcpy(ptr, res.buffer, res.length);
268
269 offset += res.length;
270 len -= res.length;
271 ptr = USB_ADD_BYTES(ptr, res.length);
272 }
273 }
274
275 /*------------------------------------------------------------------------*
276 * usbd_copy_out_user - copy directly from DMA-able memory to userland
277 *
278 * Return values:
279 * 0: Success
280 * Else: Failure
281 *------------------------------------------------------------------------*/
282 #if USB_HAVE_USER_IO
283 int
usbd_copy_out_user(struct usb_page_cache * cache,usb_frlength_t offset,void * ptr,usb_frlength_t len)284 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
285 void *ptr, usb_frlength_t len)
286 {
287 struct usb_page_search res;
288 int error;
289
290 while (len != 0) {
291
292 usbd_get_page(cache, offset, &res);
293
294 if (res.length > len) {
295 res.length = len;
296 }
297 error = copyout(res.buffer, ptr, res.length);
298 if (error)
299 return (error);
300
301 offset += res.length;
302 len -= res.length;
303 ptr = USB_ADD_BYTES(ptr, res.length);
304 }
305 return (0); /* success */
306 }
307 #endif
308
309 /*------------------------------------------------------------------------*
310 * usbd_frame_zero - zero DMA-able memory
311 *------------------------------------------------------------------------*/
312 void
usbd_frame_zero(struct usb_page_cache * cache,usb_frlength_t offset,usb_frlength_t len)313 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
314 usb_frlength_t len)
315 {
316 struct usb_page_search res;
317
318 while (len != 0) {
319
320 usbd_get_page(cache, offset, &res);
321
322 if (res.length > len) {
323 res.length = len;
324 }
325 memset(res.buffer, 0, res.length);
326
327 offset += res.length;
328 len -= res.length;
329 }
330 }
331
332 #if USB_HAVE_BUSDMA
333
334 /*------------------------------------------------------------------------*
335 * usb_dma_lock_cb - dummy callback
336 *------------------------------------------------------------------------*/
337 #if 0
338 static void
339 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
340 {
341 /* we use "mtx_owned()" instead of this function */
342 }
343 #endif
344
345 /*------------------------------------------------------------------------*
346 * usb_dma_tag_create - allocate a DMA tag
347 *
348 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
349 * allow multi-segment mappings. Else all mappings are single-segment.
350 *------------------------------------------------------------------------*/
351 static void
usb_dma_tag_create(struct usb_dma_tag * udt,usb_size_t size,usb_size_t align)352 usb_dma_tag_create(struct usb_dma_tag *udt,
353 usb_size_t size, usb_size_t align)
354 {
355 bus_dma_tag_t tag;
356
357 if (bus_dma_tag_create
358 ( /* parent */ udt->tag_parent->tag,
359 /* alignment */ align,
360 /* boundary */ 0,
361 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
362 /* highaddr */ BUS_SPACE_MAXADDR,
363 /* maxsize */ size,
364 /* nsegments */ (align == 1 && size > 1) ? (2 + (size / USB_PAGE_SIZE)) : 1,
365 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? USB_PAGE_SIZE : size,
366 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
367 &tag)) {
368 tag = NULL;
369 }
370 udt->tag = tag;
371 }
372
373 /*------------------------------------------------------------------------*
374 * usb_dma_tag_free - free a DMA tag
375 *------------------------------------------------------------------------*/
376 static void
usb_dma_tag_destroy(struct usb_dma_tag * udt)377 usb_dma_tag_destroy(struct usb_dma_tag *udt)
378 {
379 bus_dma_tag_destroy(udt->tag);
380 }
381
382 /*------------------------------------------------------------------------*
383 * usb_pc_alloc_mem_cb - BUS-DMA callback function
384 *------------------------------------------------------------------------*/
385 static void
usb_pc_alloc_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)386 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
387 int nseg, int error)
388 {
389 usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
390 }
391
392 /*------------------------------------------------------------------------*
393 * usb_pc_load_mem_cb - BUS-DMA callback function
394 *------------------------------------------------------------------------*/
395 static void
usb_pc_load_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)396 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
397 int nseg, int error)
398 {
399 usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
400 }
401
402 /*------------------------------------------------------------------------*
403 * usb_pc_common_mem_cb - BUS-DMA callback function
404 *------------------------------------------------------------------------*/
405 static void
usb_pc_common_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error,uint8_t isload)406 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
407 int nseg, int error, uint8_t isload)
408 {
409 struct usb_dma_parent_tag *uptag;
410 struct usb_page_cache *pc;
411 struct usb_page *pg;
412 usb_size_t rem;
413 bus_size_t off;
414 uint8_t owned;
415
416 pc = arg;
417 uptag = pc->tag_parent;
418
419 /*
420 * XXX There is sometimes recursive locking here.
421 * XXX We should try to find a better solution.
422 * XXX Until further the "owned" variable does
423 * XXX the trick.
424 */
425
426 if (error) {
427 goto done;
428 }
429
430 off = 0;
431 pg = pc->page_start;
432 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
433 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
434 pc->page_offset_buf = rem;
435 pc->page_offset_end += rem;
436 #ifdef USB_DEBUG
437 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
438 /*
439 * This check verifies that the physical address is correct:
440 */
441 DPRINTFN(0, "Page offset was not preserved\n");
442 error = 1;
443 goto done;
444 }
445 #endif
446 while (pc->ismultiseg) {
447 off += USB_PAGE_SIZE;
448 if (off >= (segs->ds_len + rem)) {
449 /* page crossing */
450 nseg--;
451 segs++;
452 off = 0;
453 rem = 0;
454 if (nseg == 0)
455 break;
456 }
457 pg++;
458 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
459 }
460
461 done:
462 owned = lockowned(uptag->lock);
463 if (!owned)
464 lockmgr(uptag->lock, LK_EXCLUSIVE);
465
466 uptag->dma_error = (error ? 1 : 0);
467 if (isload) {
468 (uptag->func) (uptag);
469 } else {
470 cv_broadcast(uptag->cv);
471 }
472 if (!owned)
473 lockmgr(uptag->lock, LK_RELEASE);
474 }
475
476 /*------------------------------------------------------------------------*
477 * usb_pc_alloc_mem - allocate DMA'able memory
478 *
479 * Returns:
480 * 0: Success
481 * Else: Failure
482 *------------------------------------------------------------------------*/
483 uint8_t
usb_pc_alloc_mem(struct usb_page_cache * pc,struct usb_page * pg,usb_size_t size,usb_size_t align)484 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
485 usb_size_t size, usb_size_t align)
486 {
487 struct usb_dma_parent_tag *uptag;
488 struct usb_dma_tag *utag;
489 bus_dmamap_t map;
490 void *ptr;
491 int err;
492
493 uptag = pc->tag_parent;
494
495 if (align != 1) {
496 /*
497 * The alignment must be greater or equal to the
498 * "size" else the object can be split between two
499 * memory pages and we get a problem!
500 */
501 while (align < size) {
502 align *= 2;
503 if (align == 0) {
504 goto error;
505 }
506 }
507 #if 1
508 /*
509 * XXX BUS-DMA workaround - FIXME later:
510 *
511 * We assume that that the aligment at this point of
512 * the code is greater than or equal to the size and
513 * less than two times the size, so that if we double
514 * the size, the size will be greater than the
515 * alignment.
516 *
517 * The bus-dma system has a check for "alignment"
518 * being less than "size". If that check fails we end
519 * up using contigmalloc which is page based even for
520 * small allocations. Try to avoid that to save
521 * memory, hence we sometimes to a large number of
522 * small allocations!
523 */
524 if (size <= (USB_PAGE_SIZE / 2)) {
525 size *= 2;
526 }
527 #endif
528 }
529 /* get the correct DMA tag */
530 utag = usb_dma_tag_find(uptag, size, align);
531 if (utag == NULL) {
532 goto error;
533 }
534 /* allocate memory */
535 if (bus_dmamem_alloc(
536 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
537 goto error;
538 }
539 /* setup page cache */
540 pc->buffer = ptr;
541 pc->page_start = pg;
542 pc->page_offset_buf = 0;
543 pc->page_offset_end = size;
544 pc->map = map;
545 pc->tag = utag->tag;
546 pc->ismultiseg = (align == 1);
547
548 lockmgr(uptag->lock, LK_EXCLUSIVE);
549
550 /* load memory into DMA */
551 err = bus_dmamap_load(
552 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
553 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
554
555 if (err == EINPROGRESS) {
556 cv_wait(uptag->cv, uptag->lock);
557 err = 0;
558 }
559 lockmgr(uptag->lock, LK_RELEASE);
560
561 if (err || uptag->dma_error) {
562 bus_dmamem_free(utag->tag, ptr, map);
563 goto error;
564 }
565 memset(ptr, 0, size);
566
567 usb_pc_cpu_flush(pc);
568
569 return (0);
570
571 error:
572 /* reset most of the page cache */
573 pc->buffer = NULL;
574 pc->page_start = NULL;
575 pc->page_offset_buf = 0;
576 pc->page_offset_end = 0;
577 pc->map = NULL;
578 pc->tag = NULL;
579 return (1);
580 }
581
582 /*------------------------------------------------------------------------*
583 * usb_pc_free_mem - free DMA memory
584 *
585 * This function is NULL safe.
586 *------------------------------------------------------------------------*/
587 void
usb_pc_free_mem(struct usb_page_cache * pc)588 usb_pc_free_mem(struct usb_page_cache *pc)
589 {
590 if (pc && pc->buffer) {
591
592 bus_dmamap_unload(pc->tag, pc->map);
593
594 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
595
596 pc->buffer = NULL;
597 }
598 }
599
600 /*------------------------------------------------------------------------*
601 * usb_pc_load_mem - load virtual memory into DMA
602 *
603 * Return values:
604 * 0: Success
605 * Else: Error
606 *------------------------------------------------------------------------*/
607 uint8_t
usb_pc_load_mem(struct usb_page_cache * pc,usb_size_t size,uint8_t sync)608 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
609 {
610 /* setup page cache */
611 pc->page_offset_buf = 0;
612 pc->page_offset_end = size;
613 pc->ismultiseg = 1;
614
615 KKASSERT(lockowned(pc->tag_parent->lock));
616
617 if (size > 0) {
618 if (sync) {
619 struct usb_dma_parent_tag *uptag;
620 int err;
621
622 uptag = pc->tag_parent;
623
624 /*
625 * We have to unload the previous loaded DMA
626 * pages before trying to load a new one!
627 */
628 bus_dmamap_unload(pc->tag, pc->map);
629
630 /*
631 * Try to load memory into DMA.
632 */
633 err = bus_dmamap_load(
634 pc->tag, pc->map, pc->buffer, size,
635 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
636 if (err == EINPROGRESS) {
637 cv_wait(uptag->cv, uptag->lock);
638 err = 0;
639 }
640 if (err || uptag->dma_error) {
641 return (1);
642 }
643 } else {
644
645 /*
646 * We have to unload the previous loaded DMA
647 * pages before trying to load a new one!
648 */
649 bus_dmamap_unload(pc->tag, pc->map);
650
651 /*
652 * Try to load memory into DMA. The callback
653 * will be called in all cases:
654 */
655 if (bus_dmamap_load(
656 pc->tag, pc->map, pc->buffer, size,
657 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
658 }
659 }
660 } else {
661 if (!sync) {
662 /*
663 * Call callback so that refcount is decremented
664 * properly:
665 */
666 pc->tag_parent->dma_error = 0;
667 (pc->tag_parent->func) (pc->tag_parent);
668 }
669 }
670 return (0);
671 }
672
673 /*------------------------------------------------------------------------*
674 * usb_pc_cpu_invalidate - invalidate CPU cache
675 *------------------------------------------------------------------------*/
676 void
usb_pc_cpu_invalidate(struct usb_page_cache * pc)677 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
678 {
679 if (pc->page_offset_end == pc->page_offset_buf) {
680 /* nothing has been loaded into this page cache! */
681 return;
682 }
683
684 /*
685 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
686 * same time, but in the future we should try to isolate the
687 * different cases to optimise the code. --HPS
688 */
689 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
690 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
691 }
692
693 /*------------------------------------------------------------------------*
694 * usb_pc_cpu_flush - flush CPU cache
695 *------------------------------------------------------------------------*/
696 void
usb_pc_cpu_flush(struct usb_page_cache * pc)697 usb_pc_cpu_flush(struct usb_page_cache *pc)
698 {
699 if (pc->page_offset_end == pc->page_offset_buf) {
700 /* nothing has been loaded into this page cache! */
701 return;
702 }
703 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
704 }
705
706 /*------------------------------------------------------------------------*
707 * usb_pc_dmamap_create - create a DMA map
708 *
709 * Returns:
710 * 0: Success
711 * Else: Failure
712 *------------------------------------------------------------------------*/
713 uint8_t
usb_pc_dmamap_create(struct usb_page_cache * pc,usb_size_t size)714 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
715 {
716 struct usb_xfer_root *info;
717 struct usb_dma_tag *utag;
718
719 /* get info */
720 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
721
722 /* sanity check */
723 if (info == NULL) {
724 goto error;
725 }
726 utag = usb_dma_tag_find(pc->tag_parent, size, 1);
727 if (utag == NULL) {
728 goto error;
729 }
730 /* create DMA map */
731 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
732 goto error;
733 }
734 pc->tag = utag->tag;
735 return 0; /* success */
736
737 error:
738 pc->map = NULL;
739 pc->tag = NULL;
740 return 1; /* failure */
741 }
742
743 /*------------------------------------------------------------------------*
744 * usb_pc_dmamap_destroy
745 *
746 * This function is NULL safe.
747 *------------------------------------------------------------------------*/
748 void
usb_pc_dmamap_destroy(struct usb_page_cache * pc)749 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
750 {
751 if (pc && pc->tag) {
752 bus_dmamap_destroy(pc->tag, pc->map);
753 pc->tag = NULL;
754 pc->map = NULL;
755 }
756 }
757
758 /*------------------------------------------------------------------------*
759 * usb_dma_tag_find - factored out code
760 *------------------------------------------------------------------------*/
761 struct usb_dma_tag *
usb_dma_tag_find(struct usb_dma_parent_tag * udpt,usb_size_t size,usb_size_t align)762 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
763 usb_size_t size, usb_size_t align)
764 {
765 struct usb_dma_tag *udt;
766 uint8_t nudt;
767
768 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
769 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
770
771 udt = udpt->utag_first;
772 nudt = udpt->utag_max;
773
774 while (nudt--) {
775
776 if (udt->align == 0) {
777 usb_dma_tag_create(udt, size, align);
778 if (udt->tag == NULL) {
779 return (NULL);
780 }
781 udt->align = align;
782 udt->size = size;
783 return (udt);
784 }
785 if ((udt->align == align) && (udt->size == size)) {
786 return (udt);
787 }
788 udt++;
789 }
790 return (NULL);
791 }
792
793 /*------------------------------------------------------------------------*
794 * usb_dma_tag_setup - initialise USB DMA tags
795 *------------------------------------------------------------------------*/
796 void
usb_dma_tag_setup(struct usb_dma_parent_tag * udpt,struct usb_dma_tag * udt,bus_dma_tag_t dmat,struct lock * lock,usb_dma_callback_t * func,uint8_t ndmabits,uint8_t nudt)797 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
798 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
799 struct lock *lock, usb_dma_callback_t *func,
800 uint8_t ndmabits, uint8_t nudt)
801 {
802 memset(udpt, 0, sizeof(*udpt));
803
804 /* sanity checking */
805 if ((nudt == 0) ||
806 (ndmabits == 0) ||
807 (lock == NULL)) {
808 /* something is corrupt */
809 return;
810 }
811 /* initialise condition variable */
812 cv_init(udpt->cv, "USB DMA CV");
813
814 /* store some information */
815 udpt->lock = lock;
816 udpt->func = func;
817 udpt->tag = dmat;
818 udpt->utag_first = udt;
819 udpt->utag_max = nudt;
820 udpt->dma_bits = ndmabits;
821
822 while (nudt--) {
823 memset(udt, 0, sizeof(*udt));
824 udt->tag_parent = udpt;
825 udt++;
826 }
827 }
828
829 /*------------------------------------------------------------------------*
830 * usb_bus_tag_unsetup - factored out code
831 *------------------------------------------------------------------------*/
832 void
usb_dma_tag_unsetup(struct usb_dma_parent_tag * udpt)833 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
834 {
835 struct usb_dma_tag *udt;
836 uint8_t nudt;
837
838 udt = udpt->utag_first;
839 nudt = udpt->utag_max;
840
841 while (nudt--) {
842
843 if (udt->align) {
844 /* destroy the USB DMA tag */
845 usb_dma_tag_destroy(udt);
846 udt->align = 0;
847 }
848 udt++;
849 }
850
851 if (udpt->utag_max) {
852 /* destroy the condition variable */
853 cv_destroy(udpt->cv);
854 }
855 }
856
857 /*------------------------------------------------------------------------*
858 * usb_bdma_work_loop
859 *
860 * This function handles loading of virtual buffers into DMA and is
861 * only called when "dma_refcount" is zero.
862 *------------------------------------------------------------------------*/
863 void
usb_bdma_work_loop(struct usb_xfer_queue * pq)864 usb_bdma_work_loop(struct usb_xfer_queue *pq)
865 {
866 struct usb_xfer_root *info;
867 struct usb_xfer *xfer;
868 usb_frcount_t nframes;
869
870 xfer = pq->curr;
871 info = xfer->xroot;
872
873 KKASSERT(lockowned(info->xfer_lock));
874
875 if (xfer->error) {
876 /* some error happened */
877 USB_BUS_LOCK(info->bus);
878 usbd_transfer_done(xfer, 0);
879 USB_BUS_UNLOCK(info->bus);
880 return;
881 }
882 if (!xfer->flags_int.bdma_setup) {
883 struct usb_page *pg;
884 usb_frlength_t frlength_0;
885 uint8_t isread;
886
887 xfer->flags_int.bdma_setup = 1;
888
889 /* reset BUS-DMA load state */
890
891 info->dma_error = 0;
892
893 if (xfer->flags_int.isochronous_xfr) {
894 /* only one frame buffer */
895 nframes = 1;
896 frlength_0 = xfer->sumlen;
897 } else {
898 /* can be multiple frame buffers */
899 nframes = xfer->nframes;
900 frlength_0 = xfer->frlengths[0];
901 }
902
903 /*
904 * Set DMA direction first. This is needed to
905 * select the correct cache invalidate and cache
906 * flush operations.
907 */
908 isread = USB_GET_DATA_ISREAD(xfer);
909 pg = xfer->dma_page_ptr;
910
911 if (xfer->flags_int.control_xfr &&
912 xfer->flags_int.control_hdr) {
913 /* special case */
914 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
915 /* The device controller writes to memory */
916 xfer->frbuffers[0].isread = 1;
917 } else {
918 /* The host controller reads from memory */
919 xfer->frbuffers[0].isread = 0;
920 }
921 } else {
922 /* default case */
923 xfer->frbuffers[0].isread = isread;
924 }
925
926 /*
927 * Setup the "page_start" pointer which points to an array of
928 * USB pages where information about the physical address of a
929 * page will be stored. Also initialise the "isread" field of
930 * the USB page caches.
931 */
932 xfer->frbuffers[0].page_start = pg;
933
934 info->dma_nframes = nframes;
935 info->dma_currframe = 0;
936 info->dma_frlength_0 = frlength_0;
937
938 pg += (frlength_0 / USB_PAGE_SIZE);
939 pg += 2;
940
941 while (--nframes > 0) {
942 xfer->frbuffers[nframes].isread = isread;
943 xfer->frbuffers[nframes].page_start = pg;
944
945 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
946 pg += 2;
947 }
948
949 }
950 if (info->dma_error) {
951 USB_BUS_LOCK(info->bus);
952 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
953 USB_BUS_UNLOCK(info->bus);
954 return;
955 }
956 if (info->dma_currframe != info->dma_nframes) {
957
958 if (info->dma_currframe == 0) {
959 /* special case */
960 usb_pc_load_mem(xfer->frbuffers,
961 info->dma_frlength_0, 0);
962 } else {
963 /* default case */
964 nframes = info->dma_currframe;
965 usb_pc_load_mem(xfer->frbuffers + nframes,
966 xfer->frlengths[nframes], 0);
967 }
968
969 /* advance frame index */
970 info->dma_currframe++;
971
972 return;
973 }
974 /* go ahead */
975 usb_bdma_pre_sync(xfer);
976
977 /* start loading next USB transfer, if any */
978 usb_command_wrapper(pq, NULL);
979
980 /* finally start the hardware */
981 usbd_pipe_enter(xfer);
982 }
983
984 /*------------------------------------------------------------------------*
985 * usb_bdma_done_event
986 *
987 * This function is called when the BUS-DMA has loaded virtual memory
988 * into DMA, if any.
989 *------------------------------------------------------------------------*/
990 void
usb_bdma_done_event(struct usb_dma_parent_tag * udpt)991 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
992 {
993 struct usb_xfer_root *info;
994
995 info = USB_DMATAG_TO_XROOT(udpt);
996
997 KKASSERT(lockowned(info->xfer_lock));
998
999 /* copy error */
1000 info->dma_error = udpt->dma_error;
1001
1002 /* enter workloop again */
1003 usb_command_wrapper(&info->dma_q,
1004 info->dma_q.curr);
1005 }
1006
1007 /*------------------------------------------------------------------------*
1008 * usb_bdma_pre_sync
1009 *
1010 * This function handles DMA synchronisation that must be done before
1011 * an USB transfer is started.
1012 *------------------------------------------------------------------------*/
1013 void
usb_bdma_pre_sync(struct usb_xfer * xfer)1014 usb_bdma_pre_sync(struct usb_xfer *xfer)
1015 {
1016 struct usb_page_cache *pc;
1017 usb_frcount_t nframes;
1018
1019 if (xfer->flags_int.isochronous_xfr) {
1020 /* only one frame buffer */
1021 nframes = 1;
1022 } else {
1023 /* can be multiple frame buffers */
1024 nframes = xfer->nframes;
1025 }
1026
1027 pc = xfer->frbuffers;
1028
1029 while (nframes--) {
1030
1031 if (pc->isread) {
1032 usb_pc_cpu_invalidate(pc);
1033 } else {
1034 usb_pc_cpu_flush(pc);
1035 }
1036 pc++;
1037 }
1038 }
1039
1040 /*------------------------------------------------------------------------*
1041 * usb_bdma_post_sync
1042 *
1043 * This function handles DMA synchronisation that must be done after
1044 * an USB transfer is complete.
1045 *------------------------------------------------------------------------*/
1046 void
usb_bdma_post_sync(struct usb_xfer * xfer)1047 usb_bdma_post_sync(struct usb_xfer *xfer)
1048 {
1049 struct usb_page_cache *pc;
1050 usb_frcount_t nframes;
1051
1052 if (xfer->flags_int.isochronous_xfr) {
1053 /* only one frame buffer */
1054 nframes = 1;
1055 } else {
1056 /* can be multiple frame buffers */
1057 nframes = xfer->nframes;
1058 }
1059
1060 pc = xfer->frbuffers;
1061
1062 while (nframes--) {
1063 if (pc->isread) {
1064 usb_pc_cpu_invalidate(pc);
1065 }
1066 pc++;
1067 }
1068 }
1069
1070 #endif
1071