1 /* $FreeBSD$ */
2 /*-
3 * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <bsd_global.h>
28
29 #if USB_HAVE_BUSDMA
30 static void usb_pc_common_mem_cb(struct usb_page_cache *pc,
31 void *vaddr, uint32_t length);
32 #endif
33
34 /*------------------------------------------------------------------------*
35 * usbd_get_page - lookup DMA-able memory for the given offset
36 *
37 * NOTE: Only call this function when the "page_cache" structure has
38 * been properly initialized !
39 *------------------------------------------------------------------------*/
40 void
usbd_get_page(struct usb_page_cache * pc,usb_frlength_t offset,struct usb_page_search * res)41 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
42 struct usb_page_search *res)
43 {
44 #if USB_HAVE_BUSDMA
45 struct usb_page *page;
46
47 if (pc->page_start) {
48
49 /* Case 1 - something has been loaded into DMA */
50
51 if (pc->buffer) {
52
53 /* Case 1a - Kernel Virtual Address */
54
55 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
56 }
57 offset += pc->page_offset_buf;
58
59 /* compute destination page */
60
61 page = pc->page_start;
62
63 if (pc->ismultiseg) {
64
65 page += (offset / USB_PAGE_SIZE);
66
67 offset %= USB_PAGE_SIZE;
68
69 res->length = USB_PAGE_SIZE - offset;
70 res->physaddr = page->physaddr + offset;
71 } else {
72 res->length = (usb_size_t)-1;
73 res->physaddr = page->physaddr + offset;
74 }
75 if (!pc->buffer) {
76
77 /* Case 1b - Non Kernel Virtual Address */
78
79 res->buffer = USB_ADD_BYTES(page->buffer, offset);
80 }
81 return;
82 }
83 #endif
84 /* Case 2 - Plain PIO */
85
86 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
87 res->length = (usb_size_t)-1;
88 #if USB_HAVE_BUSDMA
89 res->physaddr = 0;
90 #endif
91 }
92
93 /*------------------------------------------------------------------------*
94 * usbd_copy_in - copy directly to DMA-able memory
95 *------------------------------------------------------------------------*/
96 void
usbd_copy_in(struct usb_page_cache * cache,usb_frlength_t offset,const void * ptr,usb_frlength_t len)97 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
98 const void *ptr, usb_frlength_t len)
99 {
100 struct usb_page_search buf_res;
101
102 while (len != 0) {
103
104 usbd_get_page(cache, offset, &buf_res);
105
106 if (buf_res.length > len) {
107 buf_res.length = len;
108 }
109 memcpy(buf_res.buffer, ptr, buf_res.length);
110
111 offset += buf_res.length;
112 len -= buf_res.length;
113 ptr = USB_ADD_BYTES(ptr, buf_res.length);
114 }
115 }
116
117 /*------------------------------------------------------------------------*
118 * usbd_copy_out - copy directly from DMA-able memory
119 *------------------------------------------------------------------------*/
120 void
usbd_copy_out(struct usb_page_cache * cache,usb_frlength_t offset,void * ptr,usb_frlength_t len)121 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
122 void *ptr, usb_frlength_t len)
123 {
124 struct usb_page_search res;
125
126 while (len != 0) {
127
128 usbd_get_page(cache, offset, &res);
129
130 if (res.length > len) {
131 res.length = len;
132 }
133 memcpy(ptr, res.buffer, res.length);
134
135 offset += res.length;
136 len -= res.length;
137 ptr = USB_ADD_BYTES(ptr, res.length);
138 }
139 }
140
141 /*------------------------------------------------------------------------*
142 * usbd_frame_zero - zero DMA-able memory
143 *------------------------------------------------------------------------*/
144 void
usbd_frame_zero(struct usb_page_cache * cache,usb_frlength_t offset,usb_frlength_t len)145 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
146 usb_frlength_t len)
147 {
148 struct usb_page_search res;
149
150 while (len != 0) {
151
152 usbd_get_page(cache, offset, &res);
153
154 if (res.length > len) {
155 res.length = len;
156 }
157 memset(res.buffer, 0, res.length);
158
159 offset += res.length;
160 len -= res.length;
161 }
162 }
163
164 #if USB_HAVE_BUSDMA
165
166 /*------------------------------------------------------------------------*
167 * usb_pc_common_mem_cb - BUS-DMA callback function
168 *------------------------------------------------------------------------*/
169 static void
usb_pc_common_mem_cb(struct usb_page_cache * pc,void * vaddr,uint32_t length)170 usb_pc_common_mem_cb(struct usb_page_cache *pc,
171 void *vaddr, uint32_t length)
172 {
173 struct usb_page *pg;
174 usb_size_t rem;
175 bus_size_t off;
176 bus_addr_t phys = (uintptr_t)vaddr; /* XXX */
177 uint32_t nseg;
178
179 if (length == 0)
180 nseg = 1;
181 else
182 nseg = ((length + USB_PAGE_SIZE - 1) / USB_PAGE_SIZE);
183
184 pg = pc->page_start;
185 pg->physaddr = phys & ~(USB_PAGE_SIZE - 1);
186 rem = phys & (USB_PAGE_SIZE - 1);
187 pc->page_offset_buf = rem;
188 pc->page_offset_end += rem;
189 length += rem;
190
191 for (off = USB_PAGE_SIZE; off < length; off += USB_PAGE_SIZE) {
192 pg++;
193 pg->physaddr = (phys + off) & ~(USB_PAGE_SIZE - 1);
194 }
195 }
196
197 /*------------------------------------------------------------------------*
198 * usb_pc_alloc_mem - allocate DMA'able memory
199 *
200 * Returns:
201 * 0: Success
202 * Else: Failure
203 *------------------------------------------------------------------------*/
204 uint8_t
usb_pc_alloc_mem(struct usb_page_cache * pc,struct usb_page * pg,usb_size_t size,usb_size_t align)205 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
206 usb_size_t size, usb_size_t align)
207 {
208 void *ptr;
209 uint32_t rem;
210
211 /* allocate zeroed memory */
212
213 if (align != 1) {
214 ptr = malloc(size + align, XXX, XXX);
215 if (ptr == NULL)
216 goto error;
217
218 rem = (-((uintptr_t)ptr)) & (align - 1);
219 } else {
220 ptr = malloc(size, XXX, XXX);
221 if (ptr == NULL)
222 goto error;
223 rem = 0;
224 }
225
226 /* setup page cache */
227 pc->buffer = ((uint8_t *)ptr) + rem;
228 pc->page_start = pg;
229 pc->page_offset_buf = 0;
230 pc->page_offset_end = size;
231 pc->map = NULL;
232 pc->tag = ptr;
233 pc->ismultiseg = (align == 1);
234
235 /* compute physical address */
236 usb_pc_common_mem_cb(pc, pc->buffer, size);
237
238 usb_pc_cpu_flush(pc);
239 return (0);
240
241 error:
242 /* reset most of the page cache */
243 pc->buffer = NULL;
244 pc->page_start = NULL;
245 pc->page_offset_buf = 0;
246 pc->page_offset_end = 0;
247 pc->map = NULL;
248 pc->tag = NULL;
249 return (1);
250 }
251
252 /*------------------------------------------------------------------------*
253 * usb_pc_free_mem - free DMA memory
254 *
255 * This function is NULL safe.
256 *------------------------------------------------------------------------*/
257 void
usb_pc_free_mem(struct usb_page_cache * pc)258 usb_pc_free_mem(struct usb_page_cache *pc)
259 {
260 if (pc != NULL && pc->buffer != NULL) {
261 free(pc->tag, XXX);
262 pc->buffer = NULL;
263 }
264 }
265
266 /*------------------------------------------------------------------------*
267 * usb_pc_load_mem - load virtual memory into DMA
268 *
269 * Return values:
270 * 0: Success
271 * Else: Error
272 *------------------------------------------------------------------------*/
273 uint8_t
usb_pc_load_mem(struct usb_page_cache * pc,usb_size_t size,uint8_t sync)274 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
275 {
276 /* setup page cache */
277 pc->page_offset_buf = 0;
278 pc->page_offset_end = size;
279 pc->ismultiseg = 1;
280
281 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
282
283 if (size > 0) {
284 /* compute physical address */
285 usb_pc_common_mem_cb(pc, pc->buffer, size);
286 }
287 if (sync == 0) {
288 /*
289 * Call callback so that refcount is decremented
290 * properly:
291 */
292 pc->tag_parent->dma_error = 0;
293 (pc->tag_parent->func) (pc->tag_parent);
294 }
295 return (0);
296 }
297
298 /*------------------------------------------------------------------------*
299 * usb_pc_cpu_invalidate - invalidate CPU cache
300 *------------------------------------------------------------------------*/
301 void
usb_pc_cpu_invalidate(struct usb_page_cache * pc)302 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
303 {
304 if (pc->page_offset_end == pc->page_offset_buf) {
305 /* nothing has been loaded into this page cache! */
306 return;
307 }
308 /* NOP */
309 }
310
311 /*------------------------------------------------------------------------*
312 * usb_pc_cpu_flush - flush CPU cache
313 *------------------------------------------------------------------------*/
314 void
usb_pc_cpu_flush(struct usb_page_cache * pc)315 usb_pc_cpu_flush(struct usb_page_cache *pc)
316 {
317 if (pc->page_offset_end == pc->page_offset_buf) {
318 /* nothing has been loaded into this page cache! */
319 return;
320 }
321 /* NOP */
322 }
323
324 /*------------------------------------------------------------------------*
325 * usb_pc_dmamap_create - create a DMA map
326 *
327 * Returns:
328 * 0: Success
329 * Else: Failure
330 *------------------------------------------------------------------------*/
331 uint8_t
usb_pc_dmamap_create(struct usb_page_cache * pc,usb_size_t size)332 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
333 {
334 return (0); /* NOP, success */
335 }
336
337 /*------------------------------------------------------------------------*
338 * usb_pc_dmamap_destroy
339 *
340 * This function is NULL safe.
341 *------------------------------------------------------------------------*/
342 void
usb_pc_dmamap_destroy(struct usb_page_cache * pc)343 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
344 {
345 /* NOP */
346 }
347
348 /*------------------------------------------------------------------------*
349 * usb_dma_tag_setup - initialise USB DMA tags
350 *------------------------------------------------------------------------*/
351 void
usb_dma_tag_setup(struct usb_dma_parent_tag * udpt,struct usb_dma_tag * udt,bus_dma_tag_t dmat,struct mtx * mtx,usb_dma_callback_t * func,uint8_t ndmabits,uint8_t nudt)352 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
353 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
354 struct mtx *mtx, usb_dma_callback_t *func,
355 uint8_t ndmabits, uint8_t nudt)
356 {
357 memset(udpt, 0, sizeof(*udpt));
358
359 /* sanity checking */
360 if ((nudt == 0) ||
361 (ndmabits == 0) ||
362 (mtx == NULL)) {
363 /* something is corrupt */
364 return;
365 }
366 /* initialise condition variable */
367 cv_init(udpt->cv, "USB DMA CV");
368
369 /* store some information */
370 udpt->mtx = mtx;
371 udpt->func = func;
372 udpt->tag = dmat;
373 udpt->utag_first = udt;
374 udpt->utag_max = nudt;
375 udpt->dma_bits = ndmabits;
376
377 while (nudt--) {
378 memset(udt, 0, sizeof(*udt));
379 udt->tag_parent = udpt;
380 udt++;
381 }
382 }
383
384 /*------------------------------------------------------------------------*
385 * usb_bus_tag_unsetup - factored out code
386 *------------------------------------------------------------------------*/
387 void
usb_dma_tag_unsetup(struct usb_dma_parent_tag * udpt)388 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
389 {
390 struct usb_dma_tag *udt;
391 uint8_t nudt;
392
393 udt = udpt->utag_first;
394 nudt = udpt->utag_max;
395
396 while (nudt--) {
397 udt->align = 0;
398 udt++;
399 }
400
401 if (udpt->utag_max) {
402 /* destroy the condition variable */
403 cv_destroy(udpt->cv);
404 }
405 }
406
407 /*------------------------------------------------------------------------*
408 * usb_bdma_work_loop
409 *
410 * This function handles loading of virtual buffers into DMA and is
411 * only called when "dma_refcount" is zero.
412 *------------------------------------------------------------------------*/
413 void
usb_bdma_work_loop(struct usb_xfer_queue * pq)414 usb_bdma_work_loop(struct usb_xfer_queue *pq)
415 {
416 struct usb_xfer_root *info;
417 struct usb_xfer *xfer;
418 usb_frcount_t nframes;
419
420 xfer = pq->curr;
421 info = xfer->xroot;
422
423 mtx_assert(info->xfer_mtx, MA_OWNED);
424
425 if (xfer->error) {
426 /* some error happened */
427 USB_BUS_LOCK(info->bus);
428 usbd_transfer_done(xfer, 0);
429 USB_BUS_UNLOCK(info->bus);
430 return;
431 }
432 if (!xfer->flags_int.bdma_setup) {
433 struct usb_page *pg;
434 usb_frlength_t frlength_0;
435 uint8_t isread;
436
437 xfer->flags_int.bdma_setup = 1;
438
439 /* reset BUS-DMA load state */
440
441 info->dma_error = 0;
442
443 if (xfer->flags_int.isochronous_xfr) {
444 /* only one frame buffer */
445 nframes = 1;
446 frlength_0 = xfer->sumlen;
447 } else {
448 /* can be multiple frame buffers */
449 nframes = xfer->nframes;
450 frlength_0 = xfer->frlengths[0];
451 }
452
453 /*
454 * Set DMA direction first. This is needed to
455 * select the correct cache invalidate and cache
456 * flush operations.
457 */
458 isread = USB_GET_DATA_ISREAD(xfer);
459 pg = xfer->dma_page_ptr;
460
461 if (xfer->flags_int.control_xfr &&
462 xfer->flags_int.control_hdr) {
463 /* special case */
464 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
465 /* The device controller writes to memory */
466 xfer->frbuffers[0].isread = 1;
467 } else {
468 /* The host controller reads from memory */
469 xfer->frbuffers[0].isread = 0;
470 }
471 } else {
472 /* default case */
473 xfer->frbuffers[0].isread = isread;
474 }
475
476 /*
477 * Setup the "page_start" pointer which points to an array of
478 * USB pages where information about the physical address of a
479 * page will be stored. Also initialise the "isread" field of
480 * the USB page caches.
481 */
482 xfer->frbuffers[0].page_start = pg;
483
484 info->dma_nframes = nframes;
485 info->dma_currframe = 0;
486 info->dma_frlength_0 = frlength_0;
487
488 pg += (frlength_0 / USB_PAGE_SIZE);
489 pg += 2;
490
491 while (--nframes > 0) {
492 xfer->frbuffers[nframes].isread = isread;
493 xfer->frbuffers[nframes].page_start = pg;
494
495 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
496 pg += 2;
497 }
498
499 }
500 if (info->dma_error) {
501 USB_BUS_LOCK(info->bus);
502 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
503 USB_BUS_UNLOCK(info->bus);
504 return;
505 }
506 if (info->dma_currframe != info->dma_nframes) {
507
508 if (info->dma_currframe == 0) {
509 /* special case */
510 usb_pc_load_mem(xfer->frbuffers,
511 info->dma_frlength_0, 0);
512 } else {
513 /* default case */
514 nframes = info->dma_currframe;
515 usb_pc_load_mem(xfer->frbuffers + nframes,
516 xfer->frlengths[nframes], 0);
517 }
518
519 /* advance frame index */
520 info->dma_currframe++;
521
522 return;
523 }
524 /* go ahead */
525 usb_bdma_pre_sync(xfer);
526
527 /* start loading next USB transfer, if any */
528 usb_command_wrapper(pq, NULL);
529
530 /* finally start the hardware */
531 usbd_pipe_enter(xfer);
532 }
533
534 /*------------------------------------------------------------------------*
535 * usb_bdma_done_event
536 *
537 * This function is called when the BUS-DMA has loaded virtual memory
538 * into DMA, if any.
539 *------------------------------------------------------------------------*/
540 void
usb_bdma_done_event(struct usb_dma_parent_tag * udpt)541 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
542 {
543 struct usb_xfer_root *info;
544
545 info = USB_DMATAG_TO_XROOT(udpt);
546
547 mtx_assert(info->xfer_mtx, MA_OWNED);
548
549 /* copy error */
550 info->dma_error = udpt->dma_error;
551
552 /* enter workloop again */
553 usb_command_wrapper(&info->dma_q,
554 info->dma_q.curr);
555 }
556
557 /*------------------------------------------------------------------------*
558 * usb_bdma_pre_sync
559 *
560 * This function handles DMA synchronisation that must be done before
561 * an USB transfer is started.
562 *------------------------------------------------------------------------*/
563 void
usb_bdma_pre_sync(struct usb_xfer * xfer)564 usb_bdma_pre_sync(struct usb_xfer *xfer)
565 {
566 struct usb_page_cache *pc;
567 usb_frcount_t nframes;
568
569 if (xfer->flags_int.isochronous_xfr) {
570 /* only one frame buffer */
571 nframes = 1;
572 } else {
573 /* can be multiple frame buffers */
574 nframes = xfer->nframes;
575 }
576
577 pc = xfer->frbuffers;
578
579 while (nframes--) {
580
581 if (pc->isread) {
582 usb_pc_cpu_invalidate(pc);
583 } else {
584 usb_pc_cpu_flush(pc);
585 }
586 pc++;
587 }
588 }
589
590 /*------------------------------------------------------------------------*
591 * usb_bdma_post_sync
592 *
593 * This function handles DMA synchronisation that must be done after
594 * an USB transfer is complete.
595 *------------------------------------------------------------------------*/
596 void
usb_bdma_post_sync(struct usb_xfer * xfer)597 usb_bdma_post_sync(struct usb_xfer *xfer)
598 {
599 struct usb_page_cache *pc;
600 usb_frcount_t nframes;
601
602 if (xfer->flags_int.isochronous_xfr) {
603 /* only one frame buffer */
604 nframes = 1;
605 } else {
606 /* can be multiple frame buffers */
607 nframes = xfer->nframes;
608 }
609
610 pc = xfer->frbuffers;
611
612 while (nframes--) {
613 if (pc->isread) {
614 usb_pc_cpu_invalidate(pc);
615 }
616 pc++;
617 }
618 }
619 #endif
620