1 /*-
2  * Copyright (c) 2005 Olivier Houchard.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  */
24 
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD: stable/10/sys/arm/xscale/i80321/i80321_dma.c 236987 2012-06-13 04:38:09Z imp $");
27 #include <sys/param.h>
28 #include <sys/systm.h>
29 #include <sys/bus.h>
30 #include <sys/kernel.h>
31 #include <sys/module.h>
32 #include <sys/malloc.h>
33 #include <sys/rman.h>
34 #include <sys/lock.h>
35 #include <sys/mutex.h>
36 #include <sys/proc.h>
37 
38 #include <vm/vm.h>
39 #include <vm/pmap.h>
40 #include <vm/vm_map.h>
41 #include <machine/bus.h>
42 #include <machine/cpu.h>
43 #include <machine/md_var.h>
44 
45 #include <arm/xscale/i80321/i80321reg.h>
46 #include <arm/xscale/i80321/i80321var.h>
47 #include <arm/xscale/i80321/iq80321reg.h>
48 #include <arm/xscale/i80321/iq80321var.h>
49 #include <arm/xscale/i80321/i80321_intr.h>
50 
51 typedef struct i80321_dmadesc_s {
52 	vm_paddr_t next_desc;
53 	vm_paddr_t low_pciaddr;
54 	vm_paddr_t high_pciaddr;
55 	vm_paddr_t local_addr;
56 	vm_size_t count;
57 	uint32_t descr_ctrl;
58 	uint64_t unused;
59 } __packed	i80321_dmadesc_t;
60 
61 typedef struct i80321_dmaring_s {
62 	i80321_dmadesc_t *desc;
63 	vm_paddr_t phys_addr;
64 	bus_dmamap_t map;
65 } i80321_dmaring_t;
66 
67 #define DMA_RING_SIZE 64
68 
69 struct i80321_dma_softc {
70 	bus_space_tag_t sc_st;
71 	bus_space_handle_t sc_dma_sh;
72 	bus_dma_tag_t dmatag;
73 	i80321_dmaring_t dmaring[DMA_RING_SIZE];
74 	int flags;
75 #define BUSY	0x1
76 	int unit;
77 	struct mtx mtx;
78 };
79 
80 static int
i80321_dma_probe(device_t dev)81 i80321_dma_probe(device_t dev)
82 {
83 	device_set_desc(dev, "I80321 DMA Unit");
84 	return (0);
85 }
86 
87 static struct i80321_dma_softc *softcs[2]; /* XXX */
88 
89 static void
i80321_mapphys(void * arg,bus_dma_segment_t * segs,int nseg,int error)90 i80321_mapphys(void *arg, bus_dma_segment_t *segs, int nseg, int error)
91 {
92 	vm_paddr_t *addr = (vm_paddr_t *)arg;
93 
94 	*addr = segs->ds_addr;
95 }
96 
97 #define DMA_REG_WRITE(softc, reg, val) \
98     bus_space_write_4((softc)->sc_st, (softc)->sc_dma_sh, \
99 	(reg), (val))
100 #define DMA_REG_READ(softc, reg) \
101     bus_space_read_4((softc)->sc_st, (softc)->sc_dma_sh, \
102 	(reg))
103 
104 #define DMA_CLEAN_MASK (0x2|0x4|0x8|0x20|0x100|0x200)
105 static int dma_memcpy(void *, void *, int, int);
106 
107 static int
i80321_dma_attach(device_t dev)108 i80321_dma_attach(device_t dev)
109 {
110 	struct i80321_dma_softc *softc = device_get_softc(dev);
111 	struct i80321_softc *sc = device_get_softc(device_get_parent(dev));
112 	int unit = device_get_unit(dev);
113 	i80321_dmadesc_t *dmadescs;
114 
115 	mtx_init(&softc->mtx, "DMA engine mtx", NULL, MTX_SPIN);
116 	softc->sc_st = sc->sc_st;
117 	if (bus_space_subregion(softc->sc_st, sc->sc_sh, unit == 0 ?
118 	    VERDE_DMA_BASE0 : VERDE_DMA_BASE1, VERDE_DMA_SIZE,
119 	    &softc->sc_dma_sh) != 0)
120 		panic("%s: unable to subregion DMA registers",
121 		    device_get_name(dev));
122 	if (bus_dma_tag_create(NULL, sizeof(i80321_dmadesc_t),
123 	    0, BUS_SPACE_MAXADDR,  BUS_SPACE_MAXADDR, NULL, NULL,
124 	    DMA_RING_SIZE * sizeof(i80321_dmadesc_t), 1,
125 	    sizeof(i80321_dmadesc_t), BUS_DMA_ALLOCNOW, busdma_lock_mutex,
126 	    &Giant, &softc->dmatag))
127 		panic("Couldn't create a dma tag");
128 	DMA_REG_WRITE(softc, 0, 0);
129 	if (bus_dmamem_alloc(softc->dmatag, (void **)&dmadescs,
130     	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &softc->dmaring[0].map))
131 		panic("Couldn't alloc dma memory");
132 	for (int i = 0; i < DMA_RING_SIZE; i++) {
133 		if (i > 0)
134 			if (bus_dmamap_create(softc->dmatag, 0,
135 			    &softc->dmaring[i].map))
136 				panic("Couldn't alloc dmamap");
137 		softc->dmaring[i].desc = &dmadescs[i];
138 		bus_dmamap_load(softc->dmatag, softc->dmaring[i].map,
139 		    softc->dmaring[i].desc, sizeof(i80321_dmadesc_t),
140 		    i80321_mapphys, &softc->dmaring[i].phys_addr, 0);
141 	}
142 	softc->unit = unit;
143 	softcs[unit] = softc;
144 	_arm_memcpy = dma_memcpy;
145 	_min_memcpy_size = 1024;
146 	return (0);
147 }
148 
149 static __inline int
virt_addr_is_valid(void * addr,int len,int write,int is_kernel)150 virt_addr_is_valid(void *addr, int len, int write, int is_kernel)
151 {
152 	int to_nextpage;
153 	char tmp = 0;
154 
155 	while (len > 0) {
156 		if (write) {
157 			if (is_kernel)
158 				*(char *)addr = 0;
159 			else if (subyte(addr, 0) != 0) {
160 				return (0);
161 			}
162 		} else {
163 			if (is_kernel)
164 				badaddr_read(addr, 1, &tmp);
165 			else if (fubyte(addr) == -1) {
166 				return (0);
167 			}
168 		}
169 		to_nextpage = ((vm_offset_t)addr & ~PAGE_MASK) +
170 		    PAGE_SIZE - (vm_offset_t)addr;
171 		if (to_nextpage >= len)
172 			break;
173 		len -= to_nextpage;
174 		addr = (void *)((vm_offset_t)addr + to_nextpage);
175 	}
176 	return (1);
177 }
178 
179 static int
dma_memcpy(void * dst,void * src,int len,int flags)180 dma_memcpy(void *dst, void *src, int len, int flags)
181 {
182 	struct i80321_dma_softc *sc;
183 	i80321_dmadesc_t *desc;
184 	int ret;
185 	int csr;
186 	int descnb = 0;
187 	int tmplen = len;
188 	int to_nextpagesrc, to_nextpagedst;
189 	int min_hop;
190 	vm_paddr_t pa, pa2, tmppa;
191 	pmap_t pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
192 
193 	if (!softcs[0] || !softcs[1])
194 		return (-1);
195 	mtx_lock_spin(&softcs[0]->mtx);
196 	if (softcs[0]->flags & BUSY) {
197 		mtx_unlock_spin(&softcs[0]->mtx);
198 		mtx_lock_spin(&softcs[1]->mtx);
199 		if (softcs[1]->flags & BUSY) {
200 			mtx_unlock(&softcs[1]->mtx);
201 			return (-1);
202 		}
203 		sc = softcs[1];
204 	} else
205 		sc = softcs[0];
206 	sc->flags |= BUSY;
207 	mtx_unlock_spin(&sc->mtx);
208 	desc = sc->dmaring[0].desc;
209 	if (flags & IS_PHYSICAL) {
210 		desc->next_desc = 0;
211 		desc->low_pciaddr = (vm_paddr_t)src;
212 		desc->high_pciaddr = 0;
213 		desc->local_addr = (vm_paddr_t)dst;
214 		desc->count = len;
215 		desc->descr_ctrl = 1 << 6; /* Local memory to local memory. */
216 		bus_dmamap_sync(sc->dmatag,
217 		    sc->dmaring[0].map,
218 		    BUS_DMASYNC_PREWRITE);
219 	} else {
220 		if (!virt_addr_is_valid(dst, len, 1, !(flags & DST_IS_USER)) ||
221 		    !virt_addr_is_valid(src, len, 0, !(flags & SRC_IS_USER))) {
222 			mtx_lock_spin(&sc->mtx);
223 			sc->flags &= ~BUSY;
224 			mtx_unlock_spin(&sc->mtx);
225 			return (-1);
226 		}
227 		cpu_dcache_wb_range((vm_offset_t)src, len);
228 		if ((vm_offset_t)dst & (31))
229 			cpu_dcache_wb_range((vm_offset_t)dst & ~31, 32);
230 		if (((vm_offset_t)dst + len) & 31)
231 			cpu_dcache_wb_range(((vm_offset_t)dst + len) & ~31,
232 			    32);
233 		cpu_dcache_inv_range((vm_offset_t)dst, len);
234 		while (tmplen > 0) {
235 			pa = (flags & SRC_IS_USER) ?
236 			    pmap_extract(pmap, (vm_offset_t)src) :
237 				    vtophys(src);
238 			pa2 = (flags & DST_IS_USER) ?
239 			    pmap_extract(pmap, (vm_offset_t)dst) :
240 				    vtophys(dst);
241 			to_nextpagesrc = ((vm_offset_t)src & ~PAGE_MASK) +
242 			    PAGE_SIZE - (vm_offset_t)src;
243 			to_nextpagedst = ((vm_offset_t)dst & ~PAGE_MASK) +
244 			    PAGE_SIZE - (vm_offset_t)dst;
245 			while (to_nextpagesrc < tmplen) {
246 				tmppa = (flags & SRC_IS_USER) ?
247 				    pmap_extract(pmap, (vm_offset_t)src +
248 				    to_nextpagesrc) :
249 					    vtophys((vm_offset_t)src +
250 						to_nextpagesrc);
251 				if (tmppa != pa + to_nextpagesrc)
252 					break;
253 				to_nextpagesrc += PAGE_SIZE;
254 			}
255 			while (to_nextpagedst < tmplen) {
256 				tmppa = (flags & DST_IS_USER) ?
257 				    pmap_extract(pmap, (vm_offset_t)dst +
258 				    to_nextpagedst) :
259 					    vtophys((vm_offset_t)dst +
260 						to_nextpagedst);
261 				if (tmppa != pa2 + to_nextpagedst)
262 					break;
263 				to_nextpagedst += PAGE_SIZE;
264 			}
265 			min_hop = to_nextpagedst > to_nextpagesrc ?
266 			    to_nextpagesrc : to_nextpagedst;
267 			if (min_hop < 64) {
268 				tmplen -= min_hop;
269 				memcpy(dst, src, min_hop);
270 				cpu_dcache_wbinv_range((vm_offset_t)dst,
271 				    min_hop);
272 
273 				src = (void *)((vm_offset_t)src + min_hop);
274 				dst = (void *)((vm_offset_t)dst + min_hop);
275 				if (tmplen <= 0 && descnb > 0) {
276 					sc->dmaring[descnb - 1].desc->next_desc
277 					    = 0;
278 					bus_dmamap_sync(sc->dmatag,
279 					    sc->dmaring[descnb - 1].map,
280 					    BUS_DMASYNC_PREWRITE);
281 				}
282 				continue;
283 			}
284 			desc->low_pciaddr = pa;
285 			desc->high_pciaddr = 0;
286 			desc->local_addr = pa2;
287 			desc->count = tmplen > min_hop ? min_hop : tmplen;
288 			desc->descr_ctrl = 1 << 6;
289 			if (min_hop < tmplen) {
290 				tmplen -= min_hop;
291 				src = (void *)((vm_offset_t)src + min_hop);
292 				dst = (void *)((vm_offset_t)dst + min_hop);
293 			} else
294 				tmplen = 0;
295 			if (descnb + 1 >= DMA_RING_SIZE) {
296 				mtx_lock_spin(&sc->mtx);
297 				sc->flags &= ~BUSY;
298 				mtx_unlock_spin(&sc->mtx);
299 				return (-1);
300 			}
301 			if (tmplen > 0) {
302 				desc->next_desc = sc->dmaring[descnb + 1].
303 				    phys_addr;
304 				bus_dmamap_sync(sc->dmatag,
305 				    sc->dmaring[descnb].map,
306 				    BUS_DMASYNC_PREWRITE);
307 				desc = sc->dmaring[descnb + 1].desc;
308 				descnb++;
309 			} else {
310 				desc->next_desc = 0;
311 				bus_dmamap_sync(sc->dmatag,
312 				    sc->dmaring[descnb].map,
313 				    BUS_DMASYNC_PREWRITE);
314 			}
315 
316 		}
317 
318 	}
319 	DMA_REG_WRITE(sc, 4 /* Status register */,
320 	    DMA_REG_READ(sc, 4) | DMA_CLEAN_MASK);
321 	DMA_REG_WRITE(sc, 0x10 /* Descriptor addr */,
322 	    sc->dmaring[0].phys_addr);
323 	DMA_REG_WRITE(sc, 0 /* Control register */, 1 | 2/* Start transfer */);
324 	while ((csr = DMA_REG_READ(sc, 0x4)) & (1 << 10));
325 	/* Wait until it's done. */
326 	if (csr & 0x2e) /* error */
327 		ret = -1;
328 	else
329 		ret = 0;
330 	DMA_REG_WRITE(sc, 0, 0);
331 	mtx_lock_spin(&sc->mtx);
332 	sc->flags &= ~BUSY;
333 	mtx_unlock_spin(&sc->mtx);
334 	return (ret);
335 }
336 
337 static device_method_t i80321_dma_methods[] = {
338 	DEVMETHOD(device_probe, i80321_dma_probe),
339 	DEVMETHOD(device_attach, i80321_dma_attach),
340 	{0, 0},
341 };
342 
343 static driver_t i80321_dma_driver = {
344 	"i80321_dma",
345 	i80321_dma_methods,
346 	sizeof(struct i80321_dma_softc),
347 };
348 
349 static devclass_t i80321_dma_devclass;
350 
351 DRIVER_MODULE(i80321_dma, iq, i80321_dma_driver, i80321_dma_devclass, 0, 0);
352