68 xchan_bufs_free_reserved(xdma_channel_t *xchan)  in xchan_bufs_free_reserved()  argument
74 	for (i = 0; i < xchan->xr_num; i++) {  in xchan_bufs_free_reserved()
75 		xr = &xchan->xr_mem[i];  in xchan_bufs_free_reserved()
83 			vmem_free(xchan->vmem, xr->buf.paddr, size);  in xchan_bufs_free_reserved()
91 xchan_bufs_alloc_reserved(xdma_channel_t *xchan)  in xchan_bufs_alloc_reserved()  argument
99 	xdma = xchan->xdma;  in xchan_bufs_alloc_reserved()
101 	if (xchan->vmem == NULL)  in xchan_bufs_alloc_reserved()
104 	for (i = 0; i < xchan->xr_num; i++) {  in xchan_bufs_alloc_reserved()
105 		xr = &xchan->xr_mem[i];  in xchan_bufs_alloc_reserved()
106 		size = round_page(xchan->maxsegsize);  in xchan_bufs_alloc_reserved()
107 		if (vmem_alloc(xchan->vmem, size,  in xchan_bufs_alloc_reserved()
111 			xchan_bufs_free_reserved(xchan);  in xchan_bufs_alloc_reserved()
121 			xchan_bufs_free_reserved(xchan);  in xchan_bufs_alloc_reserved()
131 xchan_bufs_alloc_busdma(xdma_channel_t *xchan)  in xchan_bufs_alloc_busdma()  argument
138 	xdma = xchan->xdma;  in xchan_bufs_alloc_busdma()
143 	    xchan->alignment,		/* alignment */  in xchan_bufs_alloc_busdma()
144 	    xchan->boundary,		/* boundary */  in xchan_bufs_alloc_busdma()
145 	    xchan->lowaddr,		/* lowaddr */  in xchan_bufs_alloc_busdma()
146 	    xchan->highaddr,		/* highaddr */  in xchan_bufs_alloc_busdma()
148 	    xchan->maxsegsize * xchan->maxnsegs, /* maxsize */  in xchan_bufs_alloc_busdma()
149 	    xchan->maxnsegs,		/* nsegments */  in xchan_bufs_alloc_busdma()
150 	    xchan->maxsegsize,		/* maxsegsize */  in xchan_bufs_alloc_busdma()
153 	    &xchan->dma_tag_bufs);  in xchan_bufs_alloc_busdma()
160 	for (i = 0; i < xchan->xr_num; i++) {  in xchan_bufs_alloc_busdma()
161 		xr = &xchan->xr_mem[i];  in xchan_bufs_alloc_busdma()
162 		err = bus_dmamap_create(xchan->dma_tag_bufs, 0,  in xchan_bufs_alloc_busdma()
169 			bus_dma_tag_destroy(xchan->dma_tag_bufs);  in xchan_bufs_alloc_busdma()
179 xchan_bufs_alloc(xdma_channel_t *xchan)  in xchan_bufs_alloc()  argument
184 	xdma = xchan->xdma;  in xchan_bufs_alloc()
191 	if (xchan->caps & XCHAN_CAP_BUSDMA)  in xchan_bufs_alloc()
192 		ret = xchan_bufs_alloc_busdma(xchan);  in xchan_bufs_alloc()
194 		ret = xchan_bufs_alloc_reserved(xchan);  in xchan_bufs_alloc()
202 	xchan->flags |= XCHAN_BUFS_ALLOCATED;  in xchan_bufs_alloc()
208 xchan_bufs_free(xdma_channel_t *xchan)  in xchan_bufs_free()  argument
214 	if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0)  in xchan_bufs_free()
217 	if (xchan->caps & XCHAN_CAP_BUSDMA) {  in xchan_bufs_free()
218 		for (i = 0; i < xchan->xr_num; i++) {  in xchan_bufs_free()
219 			xr = &xchan->xr_mem[i];  in xchan_bufs_free()
221 			bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);  in xchan_bufs_free()
223 		bus_dma_tag_destroy(xchan->dma_tag_bufs);  in xchan_bufs_free()
225 		xchan_bufs_free_reserved(xchan);  in xchan_bufs_free()
227 	xchan->flags &= ~XCHAN_BUFS_ALLOCATED;  in xchan_bufs_free()
233 xdma_channel_free_sg(xdma_channel_t *xchan)  in xdma_channel_free_sg()  argument
236 	xchan_bufs_free(xchan);  in xdma_channel_free_sg()
237 	xchan_sglist_free(xchan);  in xdma_channel_free_sg()
238 	xchan_bank_free(xchan);  in xdma_channel_free_sg()
247 xdma_prep_sg(xdma_channel_t *xchan, uint32_t xr_num,  in xdma_prep_sg()  argument
255 	xdma = xchan->xdma;  in xdma_prep_sg()
259 	if (xchan->flags & XCHAN_CONFIGURED) {  in xdma_prep_sg()
265 	xchan->xr_num = xr_num;  in xdma_prep_sg()
266 	xchan->maxsegsize = maxsegsize;  in xdma_prep_sg()
267 	xchan->maxnsegs = maxnsegs;  in xdma_prep_sg()
268 	xchan->alignment = alignment;  in xdma_prep_sg()
269 	xchan->boundary = boundary;  in xdma_prep_sg()
270 	xchan->lowaddr = lowaddr;  in xdma_prep_sg()
271 	xchan->highaddr = highaddr;  in xdma_prep_sg()
273 	if (xchan->maxnsegs > XDMA_MAX_SEG) {  in xdma_prep_sg()
279 	xchan_bank_init(xchan);  in xdma_prep_sg()
282 	ret = xchan_sglist_alloc(xchan);  in xdma_prep_sg()
290 	if (xchan->caps & (XCHAN_CAP_BUSDMA | XCHAN_CAP_BOUNCE)) {  in xdma_prep_sg()
291 		ret = xchan_bufs_alloc(xchan);  in xdma_prep_sg()
297 			xchan_sglist_free(xchan);  in xdma_prep_sg()
298 			xchan_bank_free(xchan);  in xdma_prep_sg()
304 	xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_SG);  in xdma_prep_sg()
306 	XCHAN_LOCK(xchan);  in xdma_prep_sg()
307 	ret = XDMA_CHANNEL_PREP_SG(xdma->dma_dev, xchan);  in xdma_prep_sg()
311 		XCHAN_UNLOCK(xchan);  in xdma_prep_sg()
315 	XCHAN_UNLOCK(xchan);  in xdma_prep_sg()
321 xchan_seg_done(xdma_channel_t *xchan,  in xchan_seg_done()  argument
328 	xr = TAILQ_FIRST(&xchan->processing);  in xchan_seg_done()
337 		if (xchan->caps & XCHAN_CAP_BUSDMA) {  in xchan_seg_done()
339 				bus_dmamap_sync(xchan->dma_tag_bufs, b->map,   in xchan_seg_done()
342 				bus_dmamap_sync(xchan->dma_tag_bufs, b->map,   in xchan_seg_done()
344 			bus_dmamap_unload(xchan->dma_tag_bufs, b->map);  in xchan_seg_done()
345 		} else if (xchan->caps & XCHAN_CAP_BOUNCE) {  in xchan_seg_done()
350 		} else if (xchan->caps & XCHAN_CAP_IOMMU) {  in xchan_seg_done()
355 			xdma_iommu_remove_entry(xchan, addr);  in xchan_seg_done()
360 		QUEUE_PROC_LOCK(xchan);  in xchan_seg_done()
361 		TAILQ_REMOVE(&xchan->processing, xr, xr_next);  in xchan_seg_done()
362 		QUEUE_PROC_UNLOCK(xchan);  in xchan_seg_done()
364 		QUEUE_OUT_LOCK(xchan);  in xchan_seg_done()
365 		TAILQ_INSERT_TAIL(&xchan->queue_out, xr, xr_next);  in xchan_seg_done()
366 		QUEUE_OUT_UNLOCK(xchan);  in xchan_seg_done()
394 _xdma_load_data_busdma(xdma_channel_t *xchan, struct xdma_request *xr,  in _xdma_load_data_busdma()  argument
403 	xdma = xchan->xdma;  in _xdma_load_data_busdma()
410 		error = bus_dmamap_load_mbuf_sg(xchan->dma_tag_bufs,  in _xdma_load_data_busdma()
417 		error = bus_dmamap_load_bio(xchan->dma_tag_bufs,  in _xdma_load_data_busdma()
443 		error = bus_dmamap_load(xchan->dma_tag_bufs, xr->buf.map,  in _xdma_load_data_busdma()
472 		bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,  in _xdma_load_data_busdma()
475 		bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,  in _xdma_load_data_busdma()
482 _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,  in _xdma_load_data()  argument
493 	KASSERT(xchan->caps & (XCHAN_CAP_NOSEG | XCHAN_CAP_BOUNCE),  in _xdma_load_data()
500 		if (xchan->caps & XCHAN_CAP_BOUNCE) {  in _xdma_load_data()
505 		} else if (xchan->caps & XCHAN_CAP_IOMMU) {  in _xdma_load_data()
514 			xdma_iommu_add_entry(xchan, &va,  in _xdma_load_data()
540 xdma_load_data(xdma_channel_t *xchan,  in xdma_load_data()  argument
547 	if (xchan->caps & XCHAN_CAP_BUSDMA)  in xdma_load_data()
548 		nsegs = _xdma_load_data_busdma(xchan, xr, seg);  in xdma_load_data()
550 		nsegs = _xdma_load_data(xchan, xr, seg);  in xdma_load_data()
561 xdma_process(xdma_channel_t *xchan,  in xdma_process()  argument
574 	XCHAN_ASSERT_LOCKED(xchan);  in xdma_process()
576 	xdma = xchan->xdma;  in xdma_process()
581 	ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);  in xdma_process()
588 	TAILQ_FOREACH_SAFE(xr, &xchan->queue_in, xr_next, xr_tmp) {  in xdma_process()
591 			if ((xchan->caps & XCHAN_CAP_NOSEG) ||  in xdma_process()
592 			    (c > xchan->maxnsegs))  in xdma_process()
593 				c = xdma_mbuf_defrag(xchan, xr);  in xdma_process()
609 		if ((c + n + xchan->maxnsegs) >= XDMA_SGLIST_MAXLEN) {  in xdma_process()
614 		nsegs = xdma_load_data(xchan, xr, seg);  in xdma_process()
621 		QUEUE_IN_LOCK(xchan);  in xdma_process()
622 		TAILQ_REMOVE(&xchan->queue_in, xr, xr_next);  in xdma_process()
623 		QUEUE_IN_UNLOCK(xchan);  in xdma_process()
625 		QUEUE_PROC_LOCK(xchan);  in xdma_process()
626 		TAILQ_INSERT_TAIL(&xchan->processing, xr, xr_next);  in xdma_process()
627 		QUEUE_PROC_UNLOCK(xchan);  in xdma_process()
634 xdma_queue_submit_sg(xdma_channel_t *xchan)  in xdma_queue_submit_sg()  argument
641 	xdma = xchan->xdma;  in xdma_queue_submit_sg()
644 	XCHAN_ASSERT_LOCKED(xchan);  in xdma_queue_submit_sg()
646 	sg = xchan->sg;  in xdma_queue_submit_sg()
648 	if ((xchan->caps & (XCHAN_CAP_BOUNCE | XCHAN_CAP_BUSDMA)) &&  in xdma_queue_submit_sg()
649 	   (xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) {  in xdma_queue_submit_sg()
656 	sg_n = xdma_process(xchan, sg);  in xdma_queue_submit_sg()
661 	ret = XDMA_CHANNEL_SUBMIT_SG(xdma->dma_dev, xchan, sg, sg_n);  in xdma_queue_submit_sg()