1 /* $OpenBSD: bus_dma.c,v 1.7 2022/12/29 11:35:01 kettenis Exp $ */
2
3 /*
4 * Copyright (c) 2003-2004 Opsycon AB (www.opsycon.se / www.opsycon.com)
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
19 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 */
28 /*-
29 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
30 * All rights reserved.
31 *
32 * This code is derived from software contributed to The NetBSD Foundation
33 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
34 * NASA Ames Research Center.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 *
45 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
46 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
47 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
48 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
49 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
50 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
51 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
52 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
53 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
54 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
55 * POSSIBILITY OF SUCH DAMAGE.
56 */
57 #include <sys/param.h>
58 #include <sys/systm.h>
59 #include <sys/proc.h>
60 #include <sys/malloc.h>
61 #include <sys/mbuf.h>
62
63 #include <uvm/uvm_extern.h>
64
65 #include <machine/bus.h>
66 #include <machine/cpu.h>
67 #include <machine/cpufunc.h>
68
69 /*
70 * Common function for DMA map creation. May be called by bus-specific
71 * DMA map creation functions.
72 */
73 int
_dmamap_create(bus_dma_tag_t t,bus_size_t size,int nsegments,bus_size_t maxsegsz,bus_size_t boundary,int flags,bus_dmamap_t * dmamp)74 _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
75 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
76 {
77 struct machine_bus_dmamap *map;
78 void *mapstore;
79 size_t mapsize;
80
81 /*
82 * Allocate and initialize the DMA map. The end of the map
83 * is a variable-sized array of segments, so we allocate enough
84 * room for them in one shot.
85 *
86 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation
87 * of ALLOCNOW notifies others that we've reserved these resources,
88 * and they are not to be freed.
89 *
90 * The bus_dmamap_t includes one bus_dma_segment_t, hence
91 * the (nsegments - 1).
92 */
93 mapsize = sizeof(struct machine_bus_dmamap) +
94 (sizeof(bus_dma_segment_t) * (nsegments - 1));
95 if ((mapstore = malloc(mapsize, M_DEVBUF, (flags & BUS_DMA_NOWAIT) ?
96 (M_NOWAIT | M_ZERO) : (M_WAITOK | M_ZERO))) == NULL)
97 return (ENOMEM);
98
99 map = (struct machine_bus_dmamap *)mapstore;
100 map->_dm_size = size;
101 map->_dm_segcnt = nsegments;
102 map->_dm_maxsegsz = maxsegsz;
103 map->_dm_boundary = boundary;
104
105 *dmamp = map;
106 return (0);
107 }
108
109 /*
110 * Common function for DMA map destruction. May be called by bus-specific
111 * DMA map destruction functions.
112 */
113 void
_dmamap_destroy(bus_dma_tag_t t,bus_dmamap_t map)114 _dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
115 {
116 size_t mapsize;
117
118 mapsize = sizeof(struct machine_bus_dmamap) +
119 (sizeof(bus_dma_segment_t) * (map->_dm_segcnt - 1));
120 free(map, M_DEVBUF, mapsize);
121 }
122
123 /*
124 * Common function for loading a DMA map with a linear buffer. May
125 * be called by bus-specific DMA map load functions.
126 */
127 int
_dmamap_load(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct proc * p,int flags)128 _dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen,
129 struct proc *p, int flags)
130 {
131 paddr_t lastaddr;
132 int seg, error;
133
134 /*
135 * Make sure that on error condition we return "no valid mappings".
136 */
137 map->dm_nsegs = 0;
138 map->dm_mapsize = 0;
139
140 if (buflen > map->_dm_size)
141 return (EINVAL);
142
143 seg = 0;
144 error = (*t->_dmamap_load_buffer)(t, map, buf, buflen, p, flags,
145 &lastaddr, &seg, 1);
146 if (error == 0) {
147 map->dm_nsegs = seg + 1;
148 map->dm_mapsize = buflen;
149 }
150
151 return (error);
152 }
153
154 /*
155 * Like _bus_dmamap_load(), but for mbufs.
156 */
157 int
_dmamap_load_mbuf(bus_dma_tag_t t,bus_dmamap_t map,struct mbuf * m0,int flags)158 _dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
159 {
160 paddr_t lastaddr;
161 int seg, error, first;
162 struct mbuf *m;
163
164 /*
165 * Make sure that on error condition we return "no valid mappings".
166 */
167 map->dm_nsegs = 0;
168 map->dm_mapsize = 0;
169
170 #ifdef DIAGNOSTIC
171 if ((m0->m_flags & M_PKTHDR) == 0)
172 panic("_dmamap_load_mbuf: no packet header");
173 #endif
174
175 if (m0->m_pkthdr.len > map->_dm_size)
176 return (EINVAL);
177
178 first = 1;
179 seg = 0;
180 error = 0;
181 for (m = m0; m != NULL && error == 0; m = m->m_next) {
182 if (m->m_len == 0)
183 continue;
184 error = (*t->_dmamap_load_buffer)(t, map, m->m_data, m->m_len,
185 NULL, flags, &lastaddr, &seg, first);
186 first = 0;
187 }
188 if (error == 0) {
189 map->dm_nsegs = seg + 1;
190 map->dm_mapsize = m0->m_pkthdr.len;
191 }
192
193 return (error);
194 }
195
196 /*
197 * Like _dmamap_load(), but for uios.
198 */
199 int
_dmamap_load_uio(bus_dma_tag_t t,bus_dmamap_t map,struct uio * uio,int flags)200 _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
201 {
202 paddr_t lastaddr;
203 int seg, i, error, first;
204 bus_size_t minlen, resid;
205 struct proc *p = NULL;
206 struct iovec *iov;
207 void *addr;
208
209 /*
210 * Make sure that on error condition we return "no valid mappings".
211 */
212 map->dm_nsegs = 0;
213 map->dm_mapsize = 0;
214
215 resid = uio->uio_resid;
216 iov = uio->uio_iov;
217
218 if (uio->uio_segflg == UIO_USERSPACE) {
219 p = uio->uio_procp;
220 #ifdef DIAGNOSTIC
221 if (p == NULL)
222 panic("_dmamap_load_uio: USERSPACE but no proc");
223 #endif
224 }
225
226 first = 1;
227 seg = 0;
228 error = 0;
229 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
230 /*
231 * Now at the first iovec to load. Load each iovec
232 * until we have exhausted the residual count.
233 */
234 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
235 addr = (void *)iov[i].iov_base;
236
237 error = (*t->_dmamap_load_buffer)(t, map, addr, minlen,
238 p, flags, &lastaddr, &seg, first);
239 first = 0;
240
241 resid -= minlen;
242 }
243 if (error == 0) {
244 map->dm_nsegs = seg + 1;
245 map->dm_mapsize = uio->uio_resid;
246 }
247
248 return (error);
249 }
250
251 /*
252 * Like _dmamap_load(), but for raw memory allocated with
253 * bus_dmamem_alloc().
254 */
255 int
_dmamap_load_raw(bus_dma_tag_t t,bus_dmamap_t map,bus_dma_segment_t * segs,int nsegs,bus_size_t size,int flags)256 _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
257 int nsegs, bus_size_t size, int flags)
258 {
259 bus_addr_t paddr, baddr, bmask, lastaddr = 0;
260 bus_size_t plen, sgsize, mapsize;
261 vaddr_t vaddr;
262 int first = 1;
263 int i, seg = 0;
264
265 /*
266 * Make sure that on error condition we return "no valid mappings".
267 */
268 map->dm_mapsize = 0;
269 map->dm_nsegs = 0;
270
271 if (nsegs > map->_dm_segcnt || size > map->_dm_size)
272 return (EINVAL);
273
274 mapsize = size;
275 bmask = ~(map->_dm_boundary - 1);
276
277 /*
278 * Assume the mapping is coherent until we run into a segment
279 * that isn't.
280 */
281 map->_dm_flags = BUS_DMA_COHERENT;
282
283 for (i = 0; i < nsegs && size > 0; i++) {
284 paddr = segs[i].ds_addr;
285 vaddr = segs[i]._ds_vaddr;
286 plen = MIN(segs[i].ds_len, size);
287
288 if (!segs[i]._ds_coherent)
289 map->_dm_flags &= ~BUS_DMA_COHERENT;
290
291 while (plen > 0) {
292 /*
293 * Compute the segment size, and adjust counts.
294 */
295 sgsize = PAGE_SIZE - ((u_long)paddr & PGOFSET);
296 if (plen < sgsize)
297 sgsize = plen;
298
299 /*
300 * Make sure we don't cross any boundaries.
301 */
302 if (map->_dm_boundary > 0) {
303 baddr = (paddr + map->_dm_boundary) & bmask;
304 if (sgsize > (baddr - paddr))
305 sgsize = (baddr - paddr);
306 }
307
308 /*
309 * Insert chunk into a segment, coalescing with
310 * previous segment if possible.
311 */
312 if (first) {
313 map->dm_segs[seg].ds_addr = paddr;
314 map->dm_segs[seg].ds_len = sgsize;
315 map->dm_segs[seg]._ds_paddr = paddr;
316 map->dm_segs[seg]._ds_vaddr = vaddr;
317 first = 0;
318 } else {
319 if (paddr == lastaddr &&
320 (map->dm_segs[seg].ds_len + sgsize) <=
321 map->_dm_maxsegsz &&
322 (map->_dm_boundary == 0 ||
323 (map->dm_segs[seg].ds_addr & bmask) ==
324 (paddr & bmask)) &&
325 (t->_flags & BUS_DMA_COHERENT ||
326 (map->dm_segs[seg]._ds_vaddr +
327 map->dm_segs[seg].ds_len == vaddr)))
328 map->dm_segs[seg].ds_len += sgsize;
329 else {
330 if (++seg >= map->_dm_segcnt)
331 return (EINVAL);
332 map->dm_segs[seg].ds_addr = paddr;
333 map->dm_segs[seg].ds_len = sgsize;
334 map->dm_segs[seg]._ds_paddr = paddr;
335 map->dm_segs[seg]._ds_vaddr = vaddr;
336 }
337 }
338
339 paddr += sgsize;
340 vaddr += sgsize;
341 plen -= sgsize;
342 size -= sgsize;
343
344 lastaddr = paddr;
345 }
346 }
347
348 map->dm_mapsize = mapsize;
349 map->dm_nsegs = seg + 1;
350 return (0);
351 }
352
353 /*
354 * Common function for unloading a DMA map. May be called by
355 * bus-specific DMA map unload functions.
356 */
357 void
_dmamap_unload(bus_dma_tag_t t,bus_dmamap_t map)358 _dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
359 {
360 /*
361 * No resources to free; just mark the mappings as
362 * invalid.
363 */
364 map->dm_nsegs = 0;
365 map->dm_mapsize = 0;
366 }
367
368 static void
_dmamap_sync_segment(paddr_t pa,psize_t len,int ops)369 _dmamap_sync_segment(paddr_t pa, psize_t len, int ops)
370 {
371 switch (ops) {
372 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
373 case BUS_DMASYNC_PREREAD:
374 cpu_dcache_wbinv_range(pa, len);
375 break;
376
377 case BUS_DMASYNC_PREWRITE:
378 cpu_dcache_wb_range(pa, len);
379 break;
380
381 /*
382 * RISC-V CPUs can do speculative loads so we need to clean the cache
383 * after a DMA read to deal with any speculatively loaded cache lines.
384 * Since these can't be dirty, we can just invalidate them and don't
385 * have to worry about having to write back their contents.
386 */
387 case BUS_DMASYNC_POSTREAD:
388 case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
389 membar_sync();
390 cpu_dcache_inv_range(pa, len);
391 break;
392 }
393 }
394
395 /*
396 * Common function for DMA map synchronization. May be called
397 * by bus-specific DMA map synchronization functions.
398 */
399 void
_dmamap_sync(bus_dma_tag_t t,bus_dmamap_t map,bus_addr_t addr,bus_size_t size,int op)400 _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
401 bus_size_t size, int op)
402 {
403 int nsegs;
404 int curseg;
405
406 /*
407 * If our tag tells us that the device we are doing DMA
408 * with is coherent, make sure the write buffer is synced
409 * and return.
410 */
411 if (t->_flags & BUS_DMA_COHERENT || map->_dm_flags & BUS_DMA_COHERENT) {
412 __asm volatile ("fence iorw,iorw" ::: "memory");
413 return;
414 }
415
416 nsegs = map->dm_nsegs;
417 curseg = 0;
418
419 while (size && nsegs) {
420 paddr_t paddr;
421 bus_size_t ssize;
422
423 ssize = map->dm_segs[curseg].ds_len;
424 paddr = map->dm_segs[curseg]._ds_paddr;
425
426 if (addr != 0) {
427 if (addr >= ssize) {
428 addr -= ssize;
429 ssize = 0;
430 } else {
431 paddr += addr;
432 ssize -= addr;
433 addr = 0;
434 }
435 }
436 if (ssize > size)
437 ssize = size;
438
439 if (ssize != 0) {
440 _dmamap_sync_segment(paddr, ssize, op);
441 size -= ssize;
442 }
443 curseg++;
444 nsegs--;
445 }
446
447 if (size != 0) {
448 panic("_dmamap_sync: ran off map!");
449 }
450 }
451
452 /*
453 * Common function for DMA-safe memory allocation. May be called
454 * by bus-specific DMA memory allocation functions.
455 */
456 int
_dmamem_alloc(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags)457 _dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
458 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
459 int flags)
460 {
461 return _dmamem_alloc_range(t, size, alignment, boundary,
462 segs, nsegs, rsegs, flags, dma_constraint.ucr_low,
463 dma_constraint.ucr_high);
464 }
465
466 /*
467 * Common function for freeing DMA-safe memory. May be called by
468 * bus-specific DMA memory free functions.
469 */
470 void
_dmamem_free(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs)471 _dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
472 {
473 vm_page_t m;
474 bus_addr_t addr;
475 struct pglist mlist;
476 int curseg;
477
478 /*
479 * Build a list of pages to free back to the VM system.
480 */
481 TAILQ_INIT(&mlist);
482 for (curseg = 0; curseg < nsegs; curseg++) {
483 for (addr = segs[curseg].ds_addr;
484 addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
485 addr += PAGE_SIZE) {
486 m = PHYS_TO_VM_PAGE(addr);
487 TAILQ_INSERT_TAIL(&mlist, m, pageq);
488 }
489 }
490
491 uvm_pglistfree(&mlist);
492 }
493
494 /*
495 * Common function for mapping DMA-safe memory. May be called by
496 * bus-specific DMA memory map functions.
497 */
498 int
_dmamem_map(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,size_t size,caddr_t * kvap,int flags)499 _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
500 caddr_t *kvap, int flags)
501 {
502 vaddr_t va, sva;
503 size_t ssize;
504 bus_addr_t addr;
505 int curseg, pmap_flags, cache;
506 const struct kmem_dyn_mode *kd;
507
508 size = round_page(size);
509 kd = flags & BUS_DMA_NOWAIT ? &kd_trylock : &kd_waitok;
510 va = (vaddr_t)km_alloc(size, &kv_any, &kp_none, kd);
511 if (va == 0)
512 return (ENOMEM);
513
514 *kvap = (caddr_t)va;
515
516 sva = va;
517 ssize = size;
518 pmap_flags = PMAP_WIRED | PMAP_CANFAIL;
519 cache = PMAP_CACHE_WB;
520 if (((t->_flags & BUS_DMA_COHERENT) == 0 &&
521 (flags & BUS_DMA_COHERENT)) || (flags & BUS_DMA_NOCACHE))
522 cache = PMAP_CACHE_CI;
523 for (curseg = 0; curseg < nsegs; curseg++) {
524 segs[curseg]._ds_vaddr = va;
525 segs[curseg]._ds_coherent = !!(flags & BUS_DMA_COHERENT);
526 for (addr = segs[curseg].ds_addr;
527 addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
528 addr += NBPG, va += NBPG, size -= NBPG) {
529 if (size == 0)
530 panic("_dmamem_map: size botch");
531 pmap_kenter_cache(va, addr,
532 PROT_READ | PROT_WRITE | pmap_flags,
533 cache);
534 }
535 pmap_update(pmap_kernel());
536 }
537
538 return (0);
539 }
540
541 /*
542 * Common function for unmapping DMA-safe memory. May be called by
543 * bus-specific DMA memory unmapping functions.
544 */
545 void
_dmamem_unmap(bus_dma_tag_t t,caddr_t kva,size_t size)546 _dmamem_unmap(bus_dma_tag_t t, caddr_t kva, size_t size)
547 {
548 km_free(kva, round_page(size), &kv_any, &kp_none);
549 }
550
551 /*
552 * Common function for mmap(2)'ing DMA-safe memory. May be called by
553 * bus-specific DMA mmap(2)'ing functions.
554 */
555 paddr_t
_dmamem_mmap(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,off_t off,int prot,int flags)556 _dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off,
557 int prot, int flags)
558 {
559 int i, pmapflags = 0;
560
561 if (flags & BUS_DMA_NOCACHE)
562 pmapflags |= PMAP_NOCACHE;
563
564 for (i = 0; i < nsegs; i++) {
565 #ifdef DIAGNOSTIC
566 if (off & PGOFSET)
567 panic("_dmamem_mmap: offset unaligned");
568 if (segs[i].ds_addr & PGOFSET)
569 panic("_dmamem_mmap: segment unaligned");
570 if (segs[i].ds_len & PGOFSET)
571 panic("_dmamem_mmap: segment size not multiple"
572 " of page size");
573 #endif
574 if (off >= segs[i].ds_len) {
575 off -= segs[i].ds_len;
576 continue;
577 }
578
579 return ((segs[i].ds_addr + off) | pmapflags);
580 }
581
582 /* Page not found. */
583 return (-1);
584 }
585
586 /**********************************************************************
587 * DMA utility functions
588 **********************************************************************/
589
590 /*
591 * Utility function to load a linear buffer. lastaddrp holds state
592 * between invocations (for multiple-buffer loads). segp contains
593 * the starting segment on entrance, and the ending segment on exit.
594 * first indicates if this is the first invocation of this function.
595 */
596 int
_dmamap_load_buffer(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct proc * p,int flags,paddr_t * lastaddrp,int * segp,int first)597 _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
598 bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp,
599 int *segp, int first)
600 {
601 bus_size_t sgsize;
602 bus_addr_t lastaddr, baddr, bmask;
603 paddr_t curaddr;
604 vaddr_t vaddr = (vaddr_t)buf;
605 int seg;
606 pmap_t pmap;
607
608 if (p != NULL)
609 pmap = p->p_vmspace->vm_map.pmap;
610 else
611 pmap = pmap_kernel();
612
613 lastaddr = *lastaddrp;
614 bmask = ~(map->_dm_boundary - 1);
615 if (t->_dma_mask != 0)
616 bmask &= t->_dma_mask;
617
618 for (seg = *segp; buflen > 0; ) {
619 /*
620 * Get the physical address for this segment.
621 */
622 if (pmap_extract(pmap, vaddr, &curaddr) == FALSE)
623 panic("_dmapmap_load_buffer: pmap_extract(%p, %lx) failed!",
624 pmap, vaddr);
625
626 /*
627 * Compute the segment size, and adjust counts.
628 */
629 sgsize = NBPG - ((u_long)vaddr & PGOFSET);
630 if (buflen < sgsize)
631 sgsize = buflen;
632
633 /*
634 * Make sure we don't cross any boundaries.
635 */
636 if (map->_dm_boundary > 0) {
637 baddr = ((bus_addr_t)curaddr + map->_dm_boundary) &
638 bmask;
639 if (sgsize > (baddr - (bus_addr_t)curaddr))
640 sgsize = (baddr - (bus_addr_t)curaddr);
641 }
642
643 /*
644 * Insert chunk into a segment, coalescing with
645 * previous segment if possible.
646 */
647 if (first) {
648 map->dm_segs[seg].ds_addr = curaddr;
649 map->dm_segs[seg].ds_len = sgsize;
650 map->dm_segs[seg]._ds_paddr = curaddr;
651 map->dm_segs[seg]._ds_vaddr = vaddr;
652 first = 0;
653 } else {
654 if ((bus_addr_t)curaddr == lastaddr &&
655 (map->dm_segs[seg].ds_len + sgsize) <=
656 map->_dm_maxsegsz &&
657 (map->_dm_boundary == 0 ||
658 (map->dm_segs[seg].ds_addr & bmask) ==
659 ((bus_addr_t)curaddr & bmask)) &&
660 (t->_flags & BUS_DMA_COHERENT ||
661 (map->dm_segs[seg]._ds_vaddr +
662 map->dm_segs[seg].ds_len == vaddr)))
663 map->dm_segs[seg].ds_len += sgsize;
664 else {
665 if (++seg >= map->_dm_segcnt)
666 break;
667 map->dm_segs[seg].ds_addr = curaddr;
668 map->dm_segs[seg].ds_len = sgsize;
669 map->dm_segs[seg]._ds_paddr = curaddr;
670 map->dm_segs[seg]._ds_vaddr = vaddr;
671 }
672 }
673
674 lastaddr = (bus_addr_t)curaddr + sgsize;
675 vaddr += sgsize;
676 buflen -= sgsize;
677 }
678
679 *segp = seg;
680 *lastaddrp = lastaddr;
681
682 /*
683 * Did we fit?
684 */
685 if (buflen != 0)
686 return (EFBIG); /* XXX better return value here? */
687
688 return (0);
689 }
690
691 /*
692 * Allocate physical memory from the given physical address range.
693 * Called by DMA-safe memory allocation methods.
694 */
695 int
_dmamem_alloc_range(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags,paddr_t low,paddr_t high)696 _dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
697 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
698 int flags, paddr_t low, paddr_t high)
699 {
700 paddr_t curaddr, lastaddr;
701 vm_page_t m;
702 struct pglist mlist;
703 int curseg, error, plaflag;
704
705 /* Always round the size. */
706 size = round_page(size);
707
708 /*
709 * Allocate pages from the VM system.
710 */
711 plaflag = flags & BUS_DMA_NOWAIT ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
712 if (flags & BUS_DMA_ZERO)
713 plaflag |= UVM_PLA_ZERO;
714
715 TAILQ_INIT(&mlist);
716 error = uvm_pglistalloc(size, low, high, alignment, boundary,
717 &mlist, nsegs, plaflag);
718 if (error)
719 return (error);
720
721 /*
722 * Compute the location, size, and number of segments actually
723 * returned by the VM code.
724 */
725 m = TAILQ_FIRST(&mlist);
726 curseg = 0;
727 lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m);
728 segs[curseg].ds_len = PAGE_SIZE;
729 m = TAILQ_NEXT(m, pageq);
730
731 for (; m != NULL; m = TAILQ_NEXT(m, pageq)) {
732 curaddr = VM_PAGE_TO_PHYS(m);
733 #ifdef DIAGNOSTIC
734 if (curaddr < low || curaddr >= high) {
735 printf("vm_page_alloc_memory returned non-sensical"
736 " address 0x%lx\n", curaddr);
737 panic("_dmamem_alloc_range");
738 }
739 #endif
740 if (curaddr == (lastaddr + PAGE_SIZE))
741 segs[curseg].ds_len += PAGE_SIZE;
742 else {
743 curseg++;
744 segs[curseg].ds_addr = curaddr;
745 segs[curseg].ds_len = PAGE_SIZE;
746 }
747 lastaddr = curaddr;
748 }
749
750 *rsegs = curseg + 1;
751
752 return (0);
753 }
754