1 /*-
2 * Copyright (c) 2005 Scott Long
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #ifndef _CPU_BUS_DMA_H_
28 #define _CPU_BUS_DMA_H_
29
30 #include <machine/cpufunc.h>
31
32 /*
33 * Bus address and size types
34 */
35
36 typedef uint64_t bus_addr_t;
37 typedef uint64_t bus_size_t;
38
39 typedef uint64_t bus_space_tag_t;
40 typedef uint64_t bus_space_handle_t;
41
42 #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL
43 #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL
44 #define BUS_SPACE_MAXSIZE (64 * 1024) /* Maximum supported size */
45 #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL
46 #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL
47 #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL
48
49 #define BUS_SPACE_UNRESTRICTED (~0) /* nsegments */
50
51 /*
52 * Values for the amd64 bus space tag, not to be used directly by MI code.
53 */
54 #define X86_64_BUS_SPACE_IO 0 /* space is i/o space */
55 #define X86_64_BUS_SPACE_MEM 1 /* space is mem space */
56
57 /*
58 * Map a region of device bus space into CPU virtual address space.
59 */
60 int bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
61 bus_space_handle_t *);
62
63 /*
64 * Unmap a region of device bus space.
65 */
66 void bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t);
67
68 /*
69 * Get a new handle for a subregion of an already-mapped area of bus space.
70 */
71
72 static __inline int bus_space_subregion(bus_space_tag_t t,
73 bus_space_handle_t bsh,
74 bus_size_t offset, bus_size_t size,
75 bus_space_handle_t *nbshp);
76
77 static __inline int
bus_space_subregion(bus_space_tag_t t __unused,bus_space_handle_t bsh,bus_size_t offset,bus_size_t size __unused,bus_space_handle_t * nbshp)78 bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh,
79 bus_size_t offset, bus_size_t size __unused,
80 bus_space_handle_t *nbshp)
81 {
82
83 *nbshp = bsh + offset;
84 return (0);
85 }
86
87 static __inline void *
bus_space_kva(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)88 bus_space_kva(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset)
89 {
90 if (tag == X86_64_BUS_SPACE_IO)
91 return ((void *)0);
92 return ((void *)(handle + offset));
93 }
94
95 /*
96 * Allocate a region of memory that is accessible to devices in bus space.
97 */
98
99 int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart,
100 bus_addr_t rend, bus_size_t size, bus_size_t align,
101 bus_size_t boundary, int flags, bus_addr_t *addrp,
102 bus_space_handle_t *bshp);
103
104 /*
105 * Free a region of bus space accessible memory.
106 */
107
108 static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh,
109 bus_size_t size);
110
111 static __inline void
bus_space_free(bus_space_tag_t t __unused,bus_space_handle_t bsh __unused,bus_size_t size __unused)112 bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused,
113 bus_size_t size __unused)
114 {
115 }
116
117
118 /*
119 * Read a 1, 2, 4, or 8 byte quantity from bus space
120 * described by tag/handle/offset.
121 */
122 static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag,
123 bus_space_handle_t handle,
124 bus_size_t offset);
125
126 static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag,
127 bus_space_handle_t handle,
128 bus_size_t offset);
129
130 static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag,
131 bus_space_handle_t handle,
132 bus_size_t offset);
133
134 static __inline u_int8_t
bus_space_read_1(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)135 bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle,
136 bus_size_t offset)
137 {
138
139 if (tag == X86_64_BUS_SPACE_IO)
140 return (inb(handle + offset));
141 return (*(volatile u_int8_t *)(handle + offset));
142 }
143
144 static __inline u_int16_t
bus_space_read_2(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)145 bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle,
146 bus_size_t offset)
147 {
148
149 if (tag == X86_64_BUS_SPACE_IO)
150 return (inw(handle + offset));
151 return (*(volatile u_int16_t *)(handle + offset));
152 }
153
154 static __inline u_int32_t
bus_space_read_4(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)155 bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle,
156 bus_size_t offset)
157 {
158
159 if (tag == X86_64_BUS_SPACE_IO)
160 return (inl(handle + offset));
161 return (*(volatile u_int32_t *)(handle + offset));
162 }
163
164 #ifdef _KERNEL
165 static __inline u_int64_t
bus_space_read_8(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)166 bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
167 bus_size_t offset)
168 {
169 if (tag == X86_64_BUS_SPACE_IO)
170 panic("bus_space_read_8: illegal on I/O space");
171 return (*(volatile u_int64_t *)(handle + offset));
172 }
173 #endif
174
175 /*
176 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
177 * described by tag/handle/offset and copy into buffer provided.
178 */
179 static __inline void bus_space_read_multi_1(bus_space_tag_t tag,
180 bus_space_handle_t bsh,
181 bus_size_t offset, u_int8_t *addr,
182 size_t count);
183
184 static __inline void bus_space_read_multi_2(bus_space_tag_t tag,
185 bus_space_handle_t bsh,
186 bus_size_t offset, u_int16_t *addr,
187 size_t count);
188
189 static __inline void bus_space_read_multi_4(bus_space_tag_t tag,
190 bus_space_handle_t bsh,
191 bus_size_t offset, u_int32_t *addr,
192 size_t count);
193
194 static __inline void
bus_space_read_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t * addr,size_t count)195 bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
196 bus_size_t offset, u_int8_t *addr, size_t count)
197 {
198
199 if (tag == X86_64_BUS_SPACE_IO)
200 insb(bsh + offset, addr, count);
201 else {
202 __asm __volatile(" \n\
203 cld \n\
204 1: movb (%2),%%al \n\
205 stosb \n\
206 loop 1b" :
207 "=D" (addr), "=c" (count) :
208 "r" (bsh + offset), "0" (addr), "1" (count) :
209 "%eax", "memory");
210 }
211 }
212
213 static __inline void
bus_space_read_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t * addr,size_t count)214 bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
215 bus_size_t offset, u_int16_t *addr, size_t count)
216 {
217
218 if (tag == X86_64_BUS_SPACE_IO)
219 insw(bsh + offset, addr, count);
220 else {
221 __asm __volatile(" \n\
222 cld \n\
223 1: movw (%2),%%ax \n\
224 stosw \n\
225 loop 1b" :
226 "=D" (addr), "=c" (count) :
227 "r" (bsh + offset), "0" (addr), "1" (count) :
228 "%eax", "memory");
229 }
230 }
231
232 static __inline void
bus_space_read_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t * addr,size_t count)233 bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
234 bus_size_t offset, u_int32_t *addr, size_t count)
235 {
236
237 if (tag == X86_64_BUS_SPACE_IO)
238 insl(bsh + offset, addr, count);
239 else {
240 __asm __volatile(" \n\
241 cld \n\
242 1: movl (%2),%%eax \n\
243 stosl \n\
244 loop 1b" :
245 "=D" (addr), "=c" (count) :
246 "r" (bsh + offset), "0" (addr), "1" (count) :
247 "%eax", "memory");
248 }
249 }
250
251 #if 0 /* Cause a link error for bus_space_read_multi_8 */
252 #define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!!
253 #endif
254
255 /*
256 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
257 * described by tag/handle and starting at `offset' and copy into
258 * buffer provided.
259 */
260 static __inline void bus_space_read_region_1(bus_space_tag_t tag,
261 bus_space_handle_t bsh,
262 bus_size_t offset, u_int8_t *addr,
263 size_t count);
264
265 static __inline void bus_space_read_region_2(bus_space_tag_t tag,
266 bus_space_handle_t bsh,
267 bus_size_t offset, u_int16_t *addr,
268 size_t count);
269
270 static __inline void bus_space_read_region_4(bus_space_tag_t tag,
271 bus_space_handle_t bsh,
272 bus_size_t offset, u_int32_t *addr,
273 size_t count);
274
275
276 static __inline void
bus_space_read_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t * addr,size_t count)277 bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
278 bus_size_t offset, u_int8_t *addr, size_t count)
279 {
280
281 if (tag == X86_64_BUS_SPACE_IO) {
282 int _port_ = bsh + offset;
283 __asm __volatile(" \n\
284 cld \n\
285 1: inb %w2,%%al \n\
286 stosb \n\
287 incl %2 \n\
288 loop 1b" :
289 "=D" (addr), "=c" (count), "=d" (_port_) :
290 "0" (addr), "1" (count), "2" (_port_) :
291 "%eax", "memory", "cc");
292 } else {
293 bus_space_handle_t _port_ = bsh + offset;
294 __asm __volatile(" \n\
295 cld \n\
296 repne \n\
297 movsb" :
298 "=D" (addr), "=c" (count), "=S" (_port_) :
299 "0" (addr), "1" (count), "2" (_port_) :
300 "memory", "cc");
301 }
302 }
303
304 static __inline void
bus_space_read_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t * addr,size_t count)305 bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
306 bus_size_t offset, u_int16_t *addr, size_t count)
307 {
308
309 if (tag == X86_64_BUS_SPACE_IO) {
310 int _port_ = bsh + offset;
311 __asm __volatile(" \n\
312 cld \n\
313 1: inw %w2,%%ax \n\
314 stosw \n\
315 addl $2,%2 \n\
316 loop 1b" :
317 "=D" (addr), "=c" (count), "=d" (_port_) :
318 "0" (addr), "1" (count), "2" (_port_) :
319 "%eax", "memory", "cc");
320 } else {
321 bus_space_handle_t _port_ = bsh + offset;
322 __asm __volatile(" \n\
323 cld \n\
324 repne \n\
325 movsw" :
326 "=D" (addr), "=c" (count), "=S" (_port_) :
327 "0" (addr), "1" (count), "2" (_port_) :
328 "memory", "cc");
329 }
330 }
331
332 static __inline void
bus_space_read_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t * addr,size_t count)333 bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
334 bus_size_t offset, u_int32_t *addr, size_t count)
335 {
336
337 if (tag == X86_64_BUS_SPACE_IO) {
338 int _port_ = bsh + offset;
339 __asm __volatile(" \n\
340 cld \n\
341 1: inl %w2,%%eax \n\
342 stosl \n\
343 addl $4,%2 \n\
344 loop 1b" :
345 "=D" (addr), "=c" (count), "=d" (_port_) :
346 "0" (addr), "1" (count), "2" (_port_) :
347 "%eax", "memory", "cc");
348 } else {
349 bus_space_handle_t _port_ = bsh + offset;
350 __asm __volatile(" \n\
351 cld \n\
352 repne \n\
353 movsl" :
354 "=D" (addr), "=c" (count), "=S" (_port_) :
355 "0" (addr), "1" (count), "2" (_port_) :
356 "memory", "cc");
357 }
358 }
359
360 #if 0 /* Cause a link error for bus_space_read_region_8 */
361 #define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!!
362 #endif
363
364 /*
365 * Write the 1, 2, 4, or 8 byte value `value' to bus space
366 * described by tag/handle/offset.
367 */
368
369 static __inline void bus_space_write_1(bus_space_tag_t tag,
370 bus_space_handle_t bsh,
371 bus_size_t offset, u_int8_t value);
372
373 static __inline void bus_space_write_2(bus_space_tag_t tag,
374 bus_space_handle_t bsh,
375 bus_size_t offset, u_int16_t value);
376
377 static __inline void bus_space_write_4(bus_space_tag_t tag,
378 bus_space_handle_t bsh,
379 bus_size_t offset, u_int32_t value);
380
381 static __inline void
bus_space_write_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value)382 bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh,
383 bus_size_t offset, u_int8_t value)
384 {
385
386 if (tag == X86_64_BUS_SPACE_IO)
387 outb(bsh + offset, value);
388 else
389 *(volatile u_int8_t *)(bsh + offset) = value;
390 }
391
392 static __inline void
bus_space_write_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value)393 bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh,
394 bus_size_t offset, u_int16_t value)
395 {
396
397 if (tag == X86_64_BUS_SPACE_IO)
398 outw(bsh + offset, value);
399 else
400 *(volatile u_int16_t *)(bsh + offset) = value;
401 }
402
403 static __inline void
bus_space_write_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value)404 bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh,
405 bus_size_t offset, u_int32_t value)
406 {
407
408 if (tag == X86_64_BUS_SPACE_IO)
409 outl(bsh + offset, value);
410 else
411 *(volatile u_int32_t *)(bsh + offset) = value;
412 }
413
414 #ifdef _KERNEL
415 static __inline void
bus_space_write_8(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int64_t value)416 bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
417 bus_size_t offset, u_int64_t value)
418 {
419 if (tag == X86_64_BUS_SPACE_IO)
420 panic("bus_space_write_8: illegal on I/O space");
421 *(volatile u_int64_t *)(bsh + offset) = value;
422 }
423 #endif
424
425 /*
426 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer
427 * provided to bus space described by tag/handle/offset.
428 */
429
430 static __inline void bus_space_write_multi_1(bus_space_tag_t tag,
431 bus_space_handle_t bsh,
432 bus_size_t offset,
433 const u_int8_t *addr,
434 size_t count);
435 static __inline void bus_space_write_multi_2(bus_space_tag_t tag,
436 bus_space_handle_t bsh,
437 bus_size_t offset,
438 const u_int16_t *addr,
439 size_t count);
440
441 static __inline void bus_space_write_multi_4(bus_space_tag_t tag,
442 bus_space_handle_t bsh,
443 bus_size_t offset,
444 const u_int32_t *addr,
445 size_t count);
446
447 static __inline void
bus_space_write_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int8_t * addr,size_t count)448 bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
449 bus_size_t offset, const u_int8_t *addr, size_t count)
450 {
451
452 if (tag == X86_64_BUS_SPACE_IO)
453 outsb(bsh + offset, addr, count);
454 else {
455 __asm __volatile(" \n\
456 cld \n\
457 1: lodsb \n\
458 movb %%al,(%2) \n\
459 loop 1b" :
460 "=S" (addr), "=c" (count) :
461 "r" (bsh + offset), "0" (addr), "1" (count) :
462 "%eax", "memory", "cc");
463 }
464 }
465
466 static __inline void
bus_space_write_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int16_t * addr,size_t count)467 bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
468 bus_size_t offset, const u_int16_t *addr, size_t count)
469 {
470
471 if (tag == X86_64_BUS_SPACE_IO)
472 outsw(bsh + offset, addr, count);
473 else {
474 __asm __volatile(" \n\
475 cld \n\
476 1: lodsw \n\
477 movw %%ax,(%2) \n\
478 loop 1b" :
479 "=S" (addr), "=c" (count) :
480 "r" (bsh + offset), "0" (addr), "1" (count) :
481 "%eax", "memory", "cc");
482 }
483 }
484
485 static __inline void
bus_space_write_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int32_t * addr,size_t count)486 bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
487 bus_size_t offset, const u_int32_t *addr, size_t count)
488 {
489
490 if (tag == X86_64_BUS_SPACE_IO)
491 outsl(bsh + offset, addr, count);
492 else {
493 __asm __volatile(" \n\
494 cld \n\
495 1: lodsl \n\
496 movl %%eax,(%2) \n\
497 loop 1b" :
498 "=S" (addr), "=c" (count) :
499 "r" (bsh + offset), "0" (addr), "1" (count) :
500 "%eax", "memory", "cc");
501 }
502 }
503
504 #if 0 /* Cause a link error for bus_space_write_multi_8 */
505 #define bus_space_write_multi_8(t, h, o, a, c) \
506 !!! bus_space_write_multi_8 unimplemented !!!
507 #endif
508
509 /*
510 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided
511 * to bus space described by tag/handle starting at `offset'.
512 */
513
514 static __inline void bus_space_write_region_1(bus_space_tag_t tag,
515 bus_space_handle_t bsh,
516 bus_size_t offset,
517 const u_int8_t *addr,
518 size_t count);
519 static __inline void bus_space_write_region_2(bus_space_tag_t tag,
520 bus_space_handle_t bsh,
521 bus_size_t offset,
522 const u_int16_t *addr,
523 size_t count);
524 static __inline void bus_space_write_region_4(bus_space_tag_t tag,
525 bus_space_handle_t bsh,
526 bus_size_t offset,
527 const u_int32_t *addr,
528 size_t count);
529
530 static __inline void
bus_space_write_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int8_t * addr,size_t count)531 bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
532 bus_size_t offset, const u_int8_t *addr, size_t count)
533 {
534
535 if (tag == X86_64_BUS_SPACE_IO) {
536 int _port_ = bsh + offset;
537 __asm __volatile(" \n\
538 cld \n\
539 1: lodsb \n\
540 outb %%al,%w0 \n\
541 incl %0 \n\
542 loop 1b" :
543 "=d" (_port_), "=S" (addr), "=c" (count) :
544 "0" (_port_), "1" (addr), "2" (count) :
545 "%eax", "memory", "cc");
546 } else {
547 bus_space_handle_t _port_ = bsh + offset;
548 __asm __volatile(" \n\
549 cld \n\
550 repne \n\
551 movsb" :
552 "=D" (_port_), "=S" (addr), "=c" (count) :
553 "0" (_port_), "1" (addr), "2" (count) :
554 "memory", "cc");
555 }
556 }
557
558 static __inline void
bus_space_write_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int16_t * addr,size_t count)559 bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
560 bus_size_t offset, const u_int16_t *addr, size_t count)
561 {
562
563 if (tag == X86_64_BUS_SPACE_IO) {
564 int _port_ = bsh + offset;
565 __asm __volatile(" \n\
566 cld \n\
567 1: lodsw \n\
568 outw %%ax,%w0 \n\
569 addl $2,%0 \n\
570 loop 1b" :
571 "=d" (_port_), "=S" (addr), "=c" (count) :
572 "0" (_port_), "1" (addr), "2" (count) :
573 "%eax", "memory", "cc");
574 } else {
575 bus_space_handle_t _port_ = bsh + offset;
576 __asm __volatile(" \n\
577 cld \n\
578 repne \n\
579 movsw" :
580 "=D" (_port_), "=S" (addr), "=c" (count) :
581 "0" (_port_), "1" (addr), "2" (count) :
582 "memory", "cc");
583 }
584 }
585
586 static __inline void
bus_space_write_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int32_t * addr,size_t count)587 bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
588 bus_size_t offset, const u_int32_t *addr, size_t count)
589 {
590
591 if (tag == X86_64_BUS_SPACE_IO) {
592 int _port_ = bsh + offset;
593 __asm __volatile(" \n\
594 cld \n\
595 1: lodsl \n\
596 outl %%eax,%w0 \n\
597 addl $4,%0 \n\
598 loop 1b" :
599 "=d" (_port_), "=S" (addr), "=c" (count) :
600 "0" (_port_), "1" (addr), "2" (count) :
601 "%eax", "memory", "cc");
602 } else {
603 bus_space_handle_t _port_ = bsh + offset;
604 __asm __volatile(" \n\
605 cld \n\
606 repne \n\
607 movsl" :
608 "=D" (_port_), "=S" (addr), "=c" (count) :
609 "0" (_port_), "1" (addr), "2" (count) :
610 "memory", "cc");
611 }
612 }
613
614 #if 0 /* Cause a link error for bus_space_write_region_8 */
615 #define bus_space_write_region_8 \
616 !!! bus_space_write_region_8 unimplemented !!!
617 #endif
618
619 /*
620 * Write the 1, 2, 4, or 8 byte value `val' to bus space described
621 * by tag/handle/offset `count' times.
622 */
623
624 static __inline void bus_space_set_multi_1(bus_space_tag_t tag,
625 bus_space_handle_t bsh,
626 bus_size_t offset,
627 u_int8_t value, size_t count);
628 static __inline void bus_space_set_multi_2(bus_space_tag_t tag,
629 bus_space_handle_t bsh,
630 bus_size_t offset,
631 u_int16_t value, size_t count);
632 static __inline void bus_space_set_multi_4(bus_space_tag_t tag,
633 bus_space_handle_t bsh,
634 bus_size_t offset,
635 u_int32_t value, size_t count);
636
637 static __inline void
bus_space_set_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value,size_t count)638 bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
639 bus_size_t offset, u_int8_t value, size_t count)
640 {
641 bus_space_handle_t addr = bsh + offset;
642
643 if (tag == X86_64_BUS_SPACE_IO)
644 while (count--)
645 outb(addr, value);
646 else
647 while (count--)
648 *(volatile u_int8_t *)(addr) = value;
649 }
650
651 static __inline void
bus_space_set_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value,size_t count)652 bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
653 bus_size_t offset, u_int16_t value, size_t count)
654 {
655 bus_space_handle_t addr = bsh + offset;
656
657 if (tag == X86_64_BUS_SPACE_IO)
658 while (count--)
659 outw(addr, value);
660 else
661 while (count--)
662 *(volatile u_int16_t *)(addr) = value;
663 }
664
665 static __inline void
bus_space_set_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value,size_t count)666 bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
667 bus_size_t offset, u_int32_t value, size_t count)
668 {
669 bus_space_handle_t addr = bsh + offset;
670
671 if (tag == X86_64_BUS_SPACE_IO)
672 while (count--)
673 outl(addr, value);
674 else
675 while (count--)
676 *(volatile u_int32_t *)(addr) = value;
677 }
678
679 #if 0 /* Cause a link error for bus_space_set_multi_8 */
680 #define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!!
681 #endif
682
683 /*
684 * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described
685 * by tag/handle starting at `offset'.
686 */
687
688 static __inline void bus_space_set_region_1(bus_space_tag_t tag,
689 bus_space_handle_t bsh,
690 bus_size_t offset, u_int8_t value,
691 size_t count);
692 static __inline void bus_space_set_region_2(bus_space_tag_t tag,
693 bus_space_handle_t bsh,
694 bus_size_t offset, u_int16_t value,
695 size_t count);
696 static __inline void bus_space_set_region_4(bus_space_tag_t tag,
697 bus_space_handle_t bsh,
698 bus_size_t offset, u_int32_t value,
699 size_t count);
700
701 static __inline void
bus_space_set_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value,size_t count)702 bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
703 bus_size_t offset, u_int8_t value, size_t count)
704 {
705 bus_space_handle_t addr = bsh + offset;
706
707 if (tag == X86_64_BUS_SPACE_IO)
708 for (; count != 0; count--, addr++)
709 outb(addr, value);
710 else
711 for (; count != 0; count--, addr++)
712 *(volatile u_int8_t *)(addr) = value;
713 }
714
715 static __inline void
bus_space_set_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value,size_t count)716 bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
717 bus_size_t offset, u_int16_t value, size_t count)
718 {
719 bus_space_handle_t addr = bsh + offset;
720
721 if (tag == X86_64_BUS_SPACE_IO)
722 for (; count != 0; count--, addr += 2)
723 outw(addr, value);
724 else
725 for (; count != 0; count--, addr += 2)
726 *(volatile u_int16_t *)(addr) = value;
727 }
728
729 static __inline void
bus_space_set_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value,size_t count)730 bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
731 bus_size_t offset, u_int32_t value, size_t count)
732 {
733 bus_space_handle_t addr = bsh + offset;
734
735 if (tag == X86_64_BUS_SPACE_IO)
736 for (; count != 0; count--, addr += 4)
737 outl(addr, value);
738 else
739 for (; count != 0; count--, addr += 4)
740 *(volatile u_int32_t *)(addr) = value;
741 }
742
743 #if 0 /* Cause a link error for bus_space_set_region_8 */
744 #define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!!
745 #endif
746
747 /*
748 * Copy `count' 1, 2, 4, or 8 byte values from bus space starting
749 * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
750 */
751
752 static __inline void bus_space_copy_region_1(bus_space_tag_t tag,
753 bus_space_handle_t bsh1,
754 bus_size_t off1,
755 bus_space_handle_t bsh2,
756 bus_size_t off2, size_t count);
757
758 static __inline void bus_space_copy_region_2(bus_space_tag_t tag,
759 bus_space_handle_t bsh1,
760 bus_size_t off1,
761 bus_space_handle_t bsh2,
762 bus_size_t off2, size_t count);
763
764 static __inline void bus_space_copy_region_4(bus_space_tag_t tag,
765 bus_space_handle_t bsh1,
766 bus_size_t off1,
767 bus_space_handle_t bsh2,
768 bus_size_t off2, size_t count);
769
770 static __inline void
bus_space_copy_region_1(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)771 bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1,
772 bus_size_t off1, bus_space_handle_t bsh2,
773 bus_size_t off2, size_t count)
774 {
775 bus_space_handle_t addr1 = bsh1 + off1;
776 bus_space_handle_t addr2 = bsh2 + off2;
777
778 if (tag == X86_64_BUS_SPACE_IO) {
779 if (addr1 >= addr2) {
780 /* src after dest: copy forward */
781 for (; count != 0; count--, addr1++, addr2++)
782 outb(addr2, inb(addr1));
783 } else {
784 /* dest after src: copy backwards */
785 for (addr1 += (count - 1), addr2 += (count - 1);
786 count != 0; count--, addr1--, addr2--)
787 outb(addr2, inb(addr1));
788 }
789 } else {
790 if (addr1 >= addr2) {
791 /* src after dest: copy forward */
792 for (; count != 0; count--, addr1++, addr2++)
793 *(volatile u_int8_t *)(addr2) =
794 *(volatile u_int8_t *)(addr1);
795 } else {
796 /* dest after src: copy backwards */
797 for (addr1 += (count - 1), addr2 += (count - 1);
798 count != 0; count--, addr1--, addr2--)
799 *(volatile u_int8_t *)(addr2) =
800 *(volatile u_int8_t *)(addr1);
801 }
802 }
803 }
804
805 static __inline void
bus_space_copy_region_2(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)806 bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1,
807 bus_size_t off1, bus_space_handle_t bsh2,
808 bus_size_t off2, size_t count)
809 {
810 bus_space_handle_t addr1 = bsh1 + off1;
811 bus_space_handle_t addr2 = bsh2 + off2;
812
813 if (tag == X86_64_BUS_SPACE_IO) {
814 if (addr1 >= addr2) {
815 /* src after dest: copy forward */
816 for (; count != 0; count--, addr1 += 2, addr2 += 2)
817 outw(addr2, inw(addr1));
818 } else {
819 /* dest after src: copy backwards */
820 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
821 count != 0; count--, addr1 -= 2, addr2 -= 2)
822 outw(addr2, inw(addr1));
823 }
824 } else {
825 if (addr1 >= addr2) {
826 /* src after dest: copy forward */
827 for (; count != 0; count--, addr1 += 2, addr2 += 2)
828 *(volatile u_int16_t *)(addr2) =
829 *(volatile u_int16_t *)(addr1);
830 } else {
831 /* dest after src: copy backwards */
832 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
833 count != 0; count--, addr1 -= 2, addr2 -= 2)
834 *(volatile u_int16_t *)(addr2) =
835 *(volatile u_int16_t *)(addr1);
836 }
837 }
838 }
839
840 static __inline void
bus_space_copy_region_4(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)841 bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1,
842 bus_size_t off1, bus_space_handle_t bsh2,
843 bus_size_t off2, size_t count)
844 {
845 bus_space_handle_t addr1 = bsh1 + off1;
846 bus_space_handle_t addr2 = bsh2 + off2;
847
848 if (tag == X86_64_BUS_SPACE_IO) {
849 if (addr1 >= addr2) {
850 /* src after dest: copy forward */
851 for (; count != 0; count--, addr1 += 4, addr2 += 4)
852 outl(addr2, inl(addr1));
853 } else {
854 /* dest after src: copy backwards */
855 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
856 count != 0; count--, addr1 -= 4, addr2 -= 4)
857 outl(addr2, inl(addr1));
858 }
859 } else {
860 if (addr1 >= addr2) {
861 /* src after dest: copy forward */
862 for (; count != 0; count--, addr1 += 4, addr2 += 4)
863 *(volatile u_int32_t *)(addr2) =
864 *(volatile u_int32_t *)(addr1);
865 } else {
866 /* dest after src: copy backwards */
867 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
868 count != 0; count--, addr1 -= 4, addr2 -= 4)
869 *(volatile u_int32_t *)(addr2) =
870 *(volatile u_int32_t *)(addr1);
871 }
872 }
873 }
874
875 #if 0 /* Cause a link error for bus_space_copy_8 */
876 #define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!!
877 #endif
878
879 /*
880 * Bus read/write barrier methods.
881 *
882 * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
883 * bus_size_t offset, bus_size_t len, int flags);
884 *
885 *
886 * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than
887 * prevent reordering by the compiler; all Intel x86 processors currently
888 * retire operations outside the CPU in program order.
889 */
890 #define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
891 #define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
892
893 static __inline void
bus_space_barrier(bus_space_tag_t tag __unused,bus_space_handle_t bsh __unused,bus_size_t offset __unused,bus_size_t len __unused,int flags)894 bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused,
895 bus_size_t offset __unused, bus_size_t len __unused, int flags)
896 {
897 if (flags & BUS_SPACE_BARRIER_READ)
898 __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory");
899 else
900 __asm __volatile("" : : : "memory");
901 }
902
903 /*
904 * Stream accesses are the same as normal accesses on amd64; there are no
905 * supported bus systems with an endianess different from the host one.
906 */
907 #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o))
908 #define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o))
909 #define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o))
910
911 #define bus_space_read_multi_stream_1(t, h, o, a, c) \
912 bus_space_read_multi_1((t), (h), (o), (a), (c))
913 #define bus_space_read_multi_stream_2(t, h, o, a, c) \
914 bus_space_read_multi_2((t), (h), (o), (a), (c))
915 #define bus_space_read_multi_stream_4(t, h, o, a, c) \
916 bus_space_read_multi_4((t), (h), (o), (a), (c))
917
918 #define bus_space_write_stream_1(t, h, o, v) \
919 bus_space_write_1((t), (h), (o), (v))
920 #define bus_space_write_stream_2(t, h, o, v) \
921 bus_space_write_2((t), (h), (o), (v))
922 #define bus_space_write_stream_4(t, h, o, v) \
923 bus_space_write_4((t), (h), (o), (v))
924
925 #define bus_space_write_multi_stream_1(t, h, o, a, c) \
926 bus_space_write_multi_1((t), (h), (o), (a), (c))
927 #define bus_space_write_multi_stream_2(t, h, o, a, c) \
928 bus_space_write_multi_2((t), (h), (o), (a), (c))
929 #define bus_space_write_multi_stream_4(t, h, o, a, c) \
930 bus_space_write_multi_4((t), (h), (o), (a), (c))
931
932 #define bus_space_set_multi_stream_1(t, h, o, v, c) \
933 bus_space_set_multi_1((t), (h), (o), (v), (c))
934 #define bus_space_set_multi_stream_2(t, h, o, v, c) \
935 bus_space_set_multi_2((t), (h), (o), (v), (c))
936 #define bus_space_set_multi_stream_4(t, h, o, v, c) \
937 bus_space_set_multi_4((t), (h), (o), (v), (c))
938
939 #define bus_space_read_region_stream_1(t, h, o, a, c) \
940 bus_space_read_region_1((t), (h), (o), (a), (c))
941 #define bus_space_read_region_stream_2(t, h, o, a, c) \
942 bus_space_read_region_2((t), (h), (o), (a), (c))
943 #define bus_space_read_region_stream_4(t, h, o, a, c) \
944 bus_space_read_region_4((t), (h), (o), (a), (c))
945
946 #define bus_space_write_region_stream_1(t, h, o, a, c) \
947 bus_space_write_region_1((t), (h), (o), (a), (c))
948 #define bus_space_write_region_stream_2(t, h, o, a, c) \
949 bus_space_write_region_2((t), (h), (o), (a), (c))
950 #define bus_space_write_region_stream_4(t, h, o, a, c) \
951 bus_space_write_region_4((t), (h), (o), (a), (c))
952
953 #define bus_space_set_region_stream_1(t, h, o, v, c) \
954 bus_space_set_region_1((t), (h), (o), (v), (c))
955 #define bus_space_set_region_stream_2(t, h, o, v, c) \
956 bus_space_set_region_2((t), (h), (o), (v), (c))
957 #define bus_space_set_region_stream_4(t, h, o, v, c) \
958 bus_space_set_region_4((t), (h), (o), (v), (c))
959
960 #define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \
961 bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c))
962 #define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \
963 bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c))
964 #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \
965 bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c))
966
967 #endif /* _CPU_BUS_DMA_H_ */
968