1 /*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice unmodified, this list of conditions, and the following
13 * disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 #ifndef _LINUXKPI_LINUX_IO_H_
30 #define _LINUXKPI_LINUX_IO_H_
31
32 #include <sys/endian.h>
33 #include <sys/types.h>
34
35 #include <machine/vm.h>
36
37 #include <linux/compiler.h>
38 #include <linux/types.h>
39 #if !defined(__arm__)
40 #include <asm/set_memory.h>
41 #endif
42
43 /*
44 * XXX This is all x86 specific. It should be bus space access.
45 */
46
47 /* rmb and wmb are declared in machine/atomic.h, so should be included first. */
48 #ifndef __io_br
49 #define __io_br() __compiler_membar()
50 #endif
51
52 #ifndef __io_ar
53 #ifdef rmb
54 #define __io_ar() rmb()
55 #else
56 #define __io_ar() __compiler_membar()
57 #endif
58 #endif
59
60 #ifndef __io_bw
61 #ifdef wmb
62 #define __io_bw() wmb()
63 #else
64 #define __io_bw() __compiler_membar()
65 #endif
66 #endif
67
68 #ifndef __io_aw
69 #define __io_aw() __compiler_membar()
70 #endif
71
72 /* Access MMIO registers atomically without barriers and byte swapping. */
73
74 static inline uint8_t
__raw_readb(const volatile void * addr)75 __raw_readb(const volatile void *addr)
76 {
77 return (*(const volatile uint8_t *)addr);
78 }
79 #define __raw_readb(addr) __raw_readb(addr)
80
81 static inline void
__raw_writeb(uint8_t v,volatile void * addr)82 __raw_writeb(uint8_t v, volatile void *addr)
83 {
84 *(volatile uint8_t *)addr = v;
85 }
86 #define __raw_writeb(v, addr) __raw_writeb(v, addr)
87
88 static inline uint16_t
__raw_readw(const volatile void * addr)89 __raw_readw(const volatile void *addr)
90 {
91 return (*(const volatile uint16_t *)addr);
92 }
93 #define __raw_readw(addr) __raw_readw(addr)
94
95 static inline void
__raw_writew(uint16_t v,volatile void * addr)96 __raw_writew(uint16_t v, volatile void *addr)
97 {
98 *(volatile uint16_t *)addr = v;
99 }
100 #define __raw_writew(v, addr) __raw_writew(v, addr)
101
102 static inline uint32_t
__raw_readl(const volatile void * addr)103 __raw_readl(const volatile void *addr)
104 {
105 return (*(const volatile uint32_t *)addr);
106 }
107 #define __raw_readl(addr) __raw_readl(addr)
108
109 static inline void
__raw_writel(uint32_t v,volatile void * addr)110 __raw_writel(uint32_t v, volatile void *addr)
111 {
112 *(volatile uint32_t *)addr = v;
113 }
114 #define __raw_writel(v, addr) __raw_writel(v, addr)
115
116 #ifdef __LP64__
117 static inline uint64_t
__raw_readq(const volatile void * addr)118 __raw_readq(const volatile void *addr)
119 {
120 return (*(const volatile uint64_t *)addr);
121 }
122 #define __raw_readq(addr) __raw_readq(addr)
123
124 static inline void
__raw_writeq(uint64_t v,volatile void * addr)125 __raw_writeq(uint64_t v, volatile void *addr)
126 {
127 *(volatile uint64_t *)addr = v;
128 }
129 #define __raw_writeq(v, addr) __raw_writeq(v, addr)
130 #endif
131
132 #define mmiowb() barrier()
133
134 /* Access little-endian MMIO registers atomically with memory barriers. */
135
136 #undef readb
137 static inline uint8_t
readb(const volatile void * addr)138 readb(const volatile void *addr)
139 {
140 uint8_t v;
141
142 __io_br();
143 v = *(const volatile uint8_t *)addr;
144 __io_ar();
145 return (v);
146 }
147 #define readb(addr) readb(addr)
148
149 #undef writeb
150 static inline void
writeb(uint8_t v,volatile void * addr)151 writeb(uint8_t v, volatile void *addr)
152 {
153 __io_bw();
154 *(volatile uint8_t *)addr = v;
155 __io_aw();
156 }
157 #define writeb(v, addr) writeb(v, addr)
158
159 #undef readw
160 static inline uint16_t
readw(const volatile void * addr)161 readw(const volatile void *addr)
162 {
163 uint16_t v;
164
165 __io_br();
166 v = le16toh(__raw_readw(addr));
167 __io_ar();
168 return (v);
169 }
170 #define readw(addr) readw(addr)
171
172 #undef writew
173 static inline void
writew(uint16_t v,volatile void * addr)174 writew(uint16_t v, volatile void *addr)
175 {
176 __io_bw();
177 __raw_writew(htole16(v), addr);
178 __io_aw();
179 }
180 #define writew(v, addr) writew(v, addr)
181
182 #undef readl
183 static inline uint32_t
readl(const volatile void * addr)184 readl(const volatile void *addr)
185 {
186 uint32_t v;
187
188 __io_br();
189 v = le32toh(__raw_readl(addr));
190 __io_ar();
191 return (v);
192 }
193 #define readl(addr) readl(addr)
194
195 #undef writel
196 static inline void
writel(uint32_t v,volatile void * addr)197 writel(uint32_t v, volatile void *addr)
198 {
199 __io_bw();
200 __raw_writel(htole32(v), addr);
201 __io_aw();
202 }
203 #define writel(v, addr) writel(v, addr)
204
205 #undef readq
206 #undef writeq
207 #ifdef __LP64__
208 static inline uint64_t
readq(const volatile void * addr)209 readq(const volatile void *addr)
210 {
211 uint64_t v;
212
213 __io_br();
214 v = le64toh(__raw_readq(addr));
215 __io_ar();
216 return (v);
217 }
218 #define readq(addr) readq(addr)
219
220 static inline void
writeq(uint64_t v,volatile void * addr)221 writeq(uint64_t v, volatile void *addr)
222 {
223 __io_bw();
224 __raw_writeq(htole64(v), addr);
225 __io_aw();
226 }
227 #define writeq(v, addr) writeq(v, addr)
228 #endif
229
230 /* Access little-endian MMIO registers atomically without memory barriers. */
231
232 #undef readb_relaxed
233 static inline uint8_t
readb_relaxed(const volatile void * addr)234 readb_relaxed(const volatile void *addr)
235 {
236 return (__raw_readb(addr));
237 }
238 #define readb_relaxed(addr) readb_relaxed(addr)
239
240 #undef writeb_relaxed
241 static inline void
writeb_relaxed(uint8_t v,volatile void * addr)242 writeb_relaxed(uint8_t v, volatile void *addr)
243 {
244 __raw_writeb(v, addr);
245 }
246 #define writeb_relaxed(v, addr) writeb_relaxed(v, addr)
247
248 #undef readw_relaxed
249 static inline uint16_t
readw_relaxed(const volatile void * addr)250 readw_relaxed(const volatile void *addr)
251 {
252 return (le16toh(__raw_readw(addr)));
253 }
254 #define readw_relaxed(addr) readw_relaxed(addr)
255
256 #undef writew_relaxed
257 static inline void
writew_relaxed(uint16_t v,volatile void * addr)258 writew_relaxed(uint16_t v, volatile void *addr)
259 {
260 __raw_writew(htole16(v), addr);
261 }
262 #define writew_relaxed(v, addr) writew_relaxed(v, addr)
263
264 #undef readl_relaxed
265 static inline uint32_t
readl_relaxed(const volatile void * addr)266 readl_relaxed(const volatile void *addr)
267 {
268 return (le32toh(__raw_readl(addr)));
269 }
270 #define readl_relaxed(addr) readl_relaxed(addr)
271
272 #undef writel_relaxed
273 static inline void
writel_relaxed(uint32_t v,volatile void * addr)274 writel_relaxed(uint32_t v, volatile void *addr)
275 {
276 __raw_writel(htole32(v), addr);
277 }
278 #define writel_relaxed(v, addr) writel_relaxed(v, addr)
279
280 #undef readq_relaxed
281 #undef writeq_relaxed
282 #ifdef __LP64__
283 static inline uint64_t
readq_relaxed(const volatile void * addr)284 readq_relaxed(const volatile void *addr)
285 {
286 return (le64toh(__raw_readq(addr)));
287 }
288 #define readq_relaxed(addr) readq_relaxed(addr)
289
290 static inline void
writeq_relaxed(uint64_t v,volatile void * addr)291 writeq_relaxed(uint64_t v, volatile void *addr)
292 {
293 __raw_writeq(htole64(v), addr);
294 }
295 #define writeq_relaxed(v, addr) writeq_relaxed(v, addr)
296 #endif
297
298 /* XXX On Linux ioread and iowrite handle both MMIO and port IO. */
299
300 #undef ioread8
301 static inline uint8_t
ioread8(const volatile void * addr)302 ioread8(const volatile void *addr)
303 {
304 return (readb(addr));
305 }
306 #define ioread8(addr) ioread8(addr)
307
308 #undef ioread16
309 static inline uint16_t
ioread16(const volatile void * addr)310 ioread16(const volatile void *addr)
311 {
312 return (readw(addr));
313 }
314 #define ioread16(addr) ioread16(addr)
315
316 #undef ioread16be
317 static inline uint16_t
ioread16be(const volatile void * addr)318 ioread16be(const volatile void *addr)
319 {
320 uint16_t v;
321
322 __io_br();
323 v = (be16toh(__raw_readw(addr)));
324 __io_ar();
325
326 return (v);
327 }
328 #define ioread16be(addr) ioread16be(addr)
329
330 #undef ioread32
331 static inline uint32_t
ioread32(const volatile void * addr)332 ioread32(const volatile void *addr)
333 {
334 return (readl(addr));
335 }
336 #define ioread32(addr) ioread32(addr)
337
338 #undef ioread32be
339 static inline uint32_t
ioread32be(const volatile void * addr)340 ioread32be(const volatile void *addr)
341 {
342 uint32_t v;
343
344 __io_br();
345 v = (be32toh(__raw_readl(addr)));
346 __io_ar();
347
348 return (v);
349 }
350 #define ioread32be(addr) ioread32be(addr)
351
352 #undef iowrite8
353 static inline void
iowrite8(uint8_t v,volatile void * addr)354 iowrite8(uint8_t v, volatile void *addr)
355 {
356 writeb(v, addr);
357 }
358 #define iowrite8(v, addr) iowrite8(v, addr)
359
360 #undef iowrite16
361 static inline void
iowrite16(uint16_t v,volatile void * addr)362 iowrite16(uint16_t v, volatile void *addr)
363 {
364 writew(v, addr);
365 }
366 #define iowrite16 iowrite16
367
368 #undef iowrite32
369 static inline void
iowrite32(uint32_t v,volatile void * addr)370 iowrite32(uint32_t v, volatile void *addr)
371 {
372 writel(v, addr);
373 }
374 #define iowrite32(v, addr) iowrite32(v, addr)
375
376 #undef iowrite32be
377 static inline void
iowrite32be(uint32_t v,volatile void * addr)378 iowrite32be(uint32_t v, volatile void *addr)
379 {
380 __io_bw();
381 __raw_writel(htobe32(v), addr);
382 __io_aw();
383 }
384 #define iowrite32be(v, addr) iowrite32be(v, addr)
385
386 #if defined(__i386__) || defined(__amd64__)
387 static inline void
_outb(u_char data,u_int port)388 _outb(u_char data, u_int port)
389 {
390 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
391 }
392 #endif
393
394 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || defined(__aarch64__) || defined(__riscv)
395 void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr);
396 #else
397 #define _ioremap_attr(...) NULL
398 #endif
399
400 struct device;
401 static inline void *
devm_ioremap(struct device * dev,resource_size_t offset,resource_size_t size)402 devm_ioremap(struct device *dev, resource_size_t offset, resource_size_t size)
403 {
404 return (NULL);
405 }
406
407 #ifdef VM_MEMATTR_DEVICE
408 #define ioremap_nocache(addr, size) \
409 _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
410 #define ioremap_wt(addr, size) \
411 _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
412 #define ioremap(addr, size) \
413 _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
414 #else
415 #define ioremap_nocache(addr, size) \
416 _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE)
417 #define ioremap_wt(addr, size) \
418 _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_THROUGH)
419 #define ioremap(addr, size) \
420 _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE)
421 #endif
422 #ifdef VM_MEMATTR_WRITE_COMBINING
423 #define ioremap_wc(addr, size) \
424 _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_COMBINING)
425 #else
426 #define ioremap_wc(addr, size) ioremap_nocache(addr, size)
427 #endif
428 #define ioremap_cache(addr, size) \
429 _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_BACK)
430 void iounmap(void *addr);
431
432 #define memset_io(a, b, c) memset((a), (b), (c))
433 #define memcpy_fromio(a, b, c) memcpy((a), (b), (c))
434 #define memcpy_toio(a, b, c) memcpy((a), (b), (c))
435
436 static inline void
__iowrite32_copy(void * to,const void * from,size_t count)437 __iowrite32_copy(void *to, const void *from, size_t count)
438 {
439 const uint32_t *src;
440 uint32_t *dst;
441 int i;
442
443 for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
444 __raw_writel(*src, dst);
445 }
446
447 static inline void
__iowrite64_copy(void * to,const void * from,size_t count)448 __iowrite64_copy(void *to, const void *from, size_t count)
449 {
450 #ifdef __LP64__
451 const uint64_t *src;
452 uint64_t *dst;
453 int i;
454
455 for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
456 __raw_writeq(*src, dst);
457 #else
458 __iowrite32_copy(to, from, count * 2);
459 #endif
460 }
461
462 static inline void
__ioread32_copy(void * to,const void * from,size_t count)463 __ioread32_copy(void *to, const void *from, size_t count)
464 {
465 const uint32_t *src;
466 uint32_t *dst;
467 int i;
468
469 for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
470 *dst = __raw_readl(src);
471 }
472
473 static inline void
__ioread64_copy(void * to,const void * from,size_t count)474 __ioread64_copy(void *to, const void *from, size_t count)
475 {
476 #ifdef __LP64__
477 const uint64_t *src;
478 uint64_t *dst;
479 int i;
480
481 for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
482 *dst = __raw_readq(src);
483 #else
484 __ioread32_copy(to, from, count * 2);
485 #endif
486 }
487
488 enum {
489 MEMREMAP_WB = 1 << 0,
490 MEMREMAP_WT = 1 << 1,
491 MEMREMAP_WC = 1 << 2,
492 };
493
494 static inline void *
memremap(resource_size_t offset,size_t size,unsigned long flags)495 memremap(resource_size_t offset, size_t size, unsigned long flags)
496 {
497 void *addr = NULL;
498
499 if ((flags & MEMREMAP_WB) &&
500 (addr = ioremap_cache(offset, size)) != NULL)
501 goto done;
502 if ((flags & MEMREMAP_WT) &&
503 (addr = ioremap_wt(offset, size)) != NULL)
504 goto done;
505 if ((flags & MEMREMAP_WC) &&
506 (addr = ioremap_wc(offset, size)) != NULL)
507 goto done;
508 done:
509 return (addr);
510 }
511
512 static inline void
memunmap(void * addr)513 memunmap(void *addr)
514 {
515 /* XXX May need to check if this is RAM */
516 iounmap(addr);
517 }
518
519 #define __MTRR_ID_BASE 1
520 int lkpi_arch_phys_wc_add(unsigned long, unsigned long);
521 void lkpi_arch_phys_wc_del(int);
522 #define arch_phys_wc_add(...) lkpi_arch_phys_wc_add(__VA_ARGS__)
523 #define arch_phys_wc_del(...) lkpi_arch_phys_wc_del(__VA_ARGS__)
524 #define arch_phys_wc_index(x) \
525 (((x) < __MTRR_ID_BASE) ? -1 : ((x) - __MTRR_ID_BASE))
526
527 #if defined(__amd64__) || defined(__i386__) || defined(__aarch64__) || defined(__powerpc__) || defined(__riscv)
528 static inline int
arch_io_reserve_memtype_wc(resource_size_t start,resource_size_t size)529 arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size)
530 {
531
532 return (set_memory_wc(start, size >> PAGE_SHIFT));
533 }
534
535 static inline void
arch_io_free_memtype_wc(resource_size_t start,resource_size_t size)536 arch_io_free_memtype_wc(resource_size_t start, resource_size_t size)
537 {
538 set_memory_wb(start, size >> PAGE_SHIFT);
539 }
540 #endif
541
542 #endif /* _LINUXKPI_LINUX_IO_H_ */
543