1 /*-
2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
32 *
33 * $FreeBSD$
34 */
35
36 #ifndef _SYS_EFSYS_H
37 #define _SYS_EFSYS_H
38
39 #ifdef __cplusplus
40 extern "C" {
41 #endif
42
43 #include <sys/param.h>
44 #include <sys/bus.h>
45 #include <sys/endian.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/mbuf.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
51 #include <sys/sdt.h>
52 #include <sys/systm.h>
53
54 #include <machine/bus.h>
55 #include <machine/endian.h>
56
57 #define EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define EFSYS_USE_UINT64 1
60 #else
61 #define EFSYS_USE_UINT64 0
62 #endif
63 #define EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define EFSYS_IS_BIG_ENDIAN 1
66 #define EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define EFSYS_IS_BIG_ENDIAN 0
69 #define EFSYS_IS_LITTLE_ENDIAN 1
70 #endif
71 #include "efx_types.h"
72
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define memmove(d, s, l) bcopy(s, d, l)
76 #endif
77
78 /* FreeBSD equivalents of Solaris things */
79 #ifndef _NOTE
80 #define _NOTE(s)
81 #endif
82
83 #ifndef B_FALSE
84 #define B_FALSE FALSE
85 #endif
86 #ifndef B_TRUE
87 #define B_TRUE TRUE
88 #endif
89
90 #ifndef IS_P2ALIGNED
91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
92 #endif
93
94 #ifndef P2ROUNDUP
95 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
96 #endif
97
98 #ifndef P2ALIGN
99 #define P2ALIGN(_x, _a) ((_x) & -(_a))
100 #endif
101
102 #ifndef IS2P
103 #define ISP2(x) (((x) & ((x) - 1)) == 0)
104 #endif
105
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
107
108 #define SFXGE_USE_BUS_SPACE_8 1
109
110 #if !defined(bus_space_read_stream_8)
111
112 #define bus_space_read_stream_8(t, h, o) \
113 bus_space_read_8((t), (h), (o))
114
115 #define bus_space_write_stream_8(t, h, o, v) \
116 bus_space_write_8((t), (h), (o), (v))
117
118 #endif
119
120 #endif
121
122 #define ENOTACTIVE EINVAL
123
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
126
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
129 static __inline void
prefetch_read_many(void * addr)130 prefetch_read_many(void *addr)
131 {
132
133 __asm__(
134 "prefetcht0 (%0)"
135 :
136 : "r" (addr));
137 }
138
139 static __inline void
prefetch_read_once(void * addr)140 prefetch_read_once(void *addr)
141 {
142
143 __asm__(
144 "prefetchnta (%0)"
145 :
146 : "r" (addr));
147 }
148 #elif defined(__sparc64__)
149 static __inline void
prefetch_read_many(void * addr)150 prefetch_read_many(void *addr)
151 {
152
153 __asm__(
154 "prefetch [%0], 0"
155 :
156 : "r" (addr));
157 }
158
159 static __inline void
prefetch_read_once(void * addr)160 prefetch_read_once(void *addr)
161 {
162
163 __asm__(
164 "prefetch [%0], 1"
165 :
166 : "r" (addr));
167 }
168 #else
169 static __inline void
prefetch_read_many(void * addr)170 prefetch_read_many(void *addr)
171 {
172
173 }
174
175 static __inline void
prefetch_read_once(void * addr)176 prefetch_read_once(void *addr)
177 {
178
179 }
180 #endif
181
182 #if defined(__i386__) || defined(__amd64__)
183 #include <vm/vm.h>
184 #include <vm/pmap.h>
185 #endif
186 static __inline void
sfxge_map_mbuf_fast(bus_dma_tag_t tag,bus_dmamap_t map,struct mbuf * m,bus_dma_segment_t * seg)187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 struct mbuf *m, bus_dma_segment_t *seg)
189 {
190 #if defined(__i386__) || defined(__amd64__)
191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 seg->ds_len = m->m_len;
193 #else
194 int nsegstmp;
195
196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
197 #endif
198 }
199
200 /* Modifiers used for Windows builds */
201 #define __in
202 #define __in_opt
203 #define __in_ecount(_n)
204 #define __in_ecount_opt(_n)
205 #define __in_bcount(_n)
206 #define __in_bcount_opt(_n)
207
208 #define __out
209 #define __out_opt
210 #define __out_ecount(_n)
211 #define __out_ecount_opt(_n)
212 #define __out_bcount(_n)
213 #define __out_bcount_opt(_n)
214
215 #define __deref_out
216
217 #define __inout
218 #define __inout_opt
219 #define __inout_ecount(_n)
220 #define __inout_ecount_opt(_n)
221 #define __inout_bcount(_n)
222 #define __inout_bcount_opt(_n)
223 #define __inout_bcount_full_opt(_n)
224
225 #define __deref_out_bcount_opt(n)
226
227 #define __checkReturn
228 #define __success(_x)
229
230 #define __drv_when(_p, _c)
231
232 /* Code inclusion options */
233
234
235 #define EFSYS_OPT_NAMES 1
236
237 #define EFSYS_OPT_FALCON 0
238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
239 #define EFSYS_OPT_SIENA 1
240 #define EFSYS_OPT_HUNTINGTON 1
241 #ifdef DEBUG
242 #define EFSYS_OPT_CHECK_REG 1
243 #else
244 #define EFSYS_OPT_CHECK_REG 0
245 #endif
246
247 #define EFSYS_OPT_MCDI 1
248 #define EFSYS_OPT_MCDI_LOGGING 0
249 #define EFSYS_OPT_MCDI_PROXY_AUTH 0
250
251 #define EFSYS_OPT_MAC_FALCON_GMAC 0
252 #define EFSYS_OPT_MAC_FALCON_XMAC 0
253 #define EFSYS_OPT_MAC_STATS 1
254
255 #define EFSYS_OPT_LOOPBACK 0
256
257 #define EFSYS_OPT_MON_NULL 0
258 #define EFSYS_OPT_MON_LM87 0
259 #define EFSYS_OPT_MON_MAX6647 0
260 #define EFSYS_OPT_MON_MCDI 0
261 #define EFSYS_OPT_MON_STATS 0
262
263 #define EFSYS_OPT_PHY_NULL 0
264 #define EFSYS_OPT_PHY_QT2022C2 0
265 #define EFSYS_OPT_PHY_SFX7101 0
266 #define EFSYS_OPT_PHY_TXC43128 0
267 #define EFSYS_OPT_PHY_SFT9001 0
268 #define EFSYS_OPT_PHY_QT2025C 0
269 #define EFSYS_OPT_PHY_STATS 1
270 #define EFSYS_OPT_PHY_PROPS 0
271 #define EFSYS_OPT_PHY_BIST 0
272 #define EFSYS_OPT_BIST 1
273 #define EFSYS_OPT_PHY_LED_CONTROL 1
274 #define EFSYS_OPT_PHY_FLAGS 0
275
276 #define EFSYS_OPT_VPD 1
277 #define EFSYS_OPT_NVRAM 1
278 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
279 #define EFSYS_OPT_NVRAM_SFT9001 0
280 #define EFSYS_OPT_NVRAM_SFX7101 0
281 #define EFSYS_OPT_BOOTCFG 0
282
283 #define EFSYS_OPT_PCIE_TUNE 0
284 #define EFSYS_OPT_DIAG 0
285 #define EFSYS_OPT_WOL 1
286 #define EFSYS_OPT_RX_SCALE 1
287 #define EFSYS_OPT_QSTATS 1
288 #define EFSYS_OPT_FILTER 1
289 #define EFSYS_OPT_MCAST_FILTER_LIST 1
290 #define EFSYS_OPT_RX_SCATTER 0
291 #define EFSYS_OPT_RX_HDR_SPLIT 0
292
293 #define EFSYS_OPT_EV_PREFETCH 0
294
295 #define EFSYS_OPT_DECODE_INTR_FATAL 1
296
297 /* ID */
298
299 typedef struct __efsys_identifier_s efsys_identifier_t;
300
301 /* PROBE */
302
303 #ifndef DTRACE_PROBE
304
305 #define EFSYS_PROBE(_name)
306
307 #define EFSYS_PROBE1(_name, _type1, _arg1)
308
309 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
310
311 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
312 _type3, _arg3)
313
314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
315 _type3, _arg3, _type4, _arg4)
316
317 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
318 _type3, _arg3, _type4, _arg4, _type5, _arg5)
319
320 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
321 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
322 _type6, _arg6)
323
324 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
325 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
326 _type6, _arg6, _type7, _arg7)
327
328 #else /* DTRACE_PROBE */
329
330 #define EFSYS_PROBE(_name) \
331 DTRACE_PROBE(_name)
332
333 #define EFSYS_PROBE1(_name, _type1, _arg1) \
334 DTRACE_PROBE1(_name, _type1, _arg1)
335
336 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
337 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
338
339 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
340 _type3, _arg3) \
341 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
342 _type3, _arg3)
343
344 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
345 _type3, _arg3, _type4, _arg4) \
346 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
347 _type3, _arg3, _type4, _arg4)
348
349 #ifdef DTRACE_PROBE5
350 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
351 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
352 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
353 _type3, _arg3, _type4, _arg4, _type5, _arg5)
354 #else
355 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
356 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
357 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
358 _type3, _arg3, _type4, _arg4)
359 #endif
360
361 #ifdef DTRACE_PROBE6
362 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
363 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
364 _type6, _arg6) \
365 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
366 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
367 _type6, _arg6)
368 #else
369 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
370 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
371 _type6, _arg6) \
372 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
373 _type3, _arg3, _type4, _arg4, _type5, _arg5)
374 #endif
375
376 #ifdef DTRACE_PROBE7
377 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
378 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
379 _type6, _arg6, _type7, _arg7) \
380 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
381 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
382 _type6, _arg6, _type7, _arg7)
383 #else
384 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
385 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
386 _type6, _arg6, _type7, _arg7) \
387 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
388 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
389 _type6, _arg6)
390 #endif
391
392 #endif /* DTRACE_PROBE */
393
394 /* DMA */
395
396 typedef uint64_t efsys_dma_addr_t;
397
398 typedef struct efsys_mem_s {
399 bus_dma_tag_t esm_tag;
400 bus_dmamap_t esm_map;
401 caddr_t esm_base;
402 efsys_dma_addr_t esm_addr;
403 } efsys_mem_t;
404
405
406 #define EFSYS_MEM_ZERO(_esmp, _size) \
407 do { \
408 (void) memset((_esmp)->esm_base, 0, (_size)); \
409 \
410 _NOTE(CONSTANTCONDITION) \
411 } while (B_FALSE)
412
413 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
414 do { \
415 uint32_t *addr; \
416 \
417 _NOTE(CONSTANTCONDITION) \
418 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
419 ("not power of 2 aligned")); \
420 \
421 addr = (void *)((_esmp)->esm_base + (_offset)); \
422 \
423 (_edp)->ed_u32[0] = *addr; \
424 \
425 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
426 uint32_t, (_edp)->ed_u32[0]); \
427 \
428 _NOTE(CONSTANTCONDITION) \
429 } while (B_FALSE)
430
431 #if defined(__x86_64__)
432 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
433 do { \
434 uint64_t *addr; \
435 \
436 _NOTE(CONSTANTCONDITION) \
437 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
438 ("not power of 2 aligned")); \
439 \
440 addr = (void *)((_esmp)->esm_base + (_offset)); \
441 \
442 (_eqp)->eq_u64[0] = *addr; \
443 \
444 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
445 uint32_t, (_eqp)->eq_u32[1], \
446 uint32_t, (_eqp)->eq_u32[0]); \
447 \
448 _NOTE(CONSTANTCONDITION) \
449 } while (B_FALSE)
450 #else
451 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
452 do { \
453 uint32_t *addr; \
454 \
455 _NOTE(CONSTANTCONDITION) \
456 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
457 ("not power of 2 aligned")); \
458 \
459 addr = (void *)((_esmp)->esm_base + (_offset)); \
460 \
461 (_eqp)->eq_u32[0] = *addr++; \
462 (_eqp)->eq_u32[1] = *addr; \
463 \
464 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
465 uint32_t, (_eqp)->eq_u32[1], \
466 uint32_t, (_eqp)->eq_u32[0]); \
467 \
468 _NOTE(CONSTANTCONDITION) \
469 } while (B_FALSE)
470 #endif
471
472 #if defined(__x86_64__)
473 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
474 do { \
475 uint64_t *addr; \
476 \
477 _NOTE(CONSTANTCONDITION) \
478 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
479 ("not power of 2 aligned")); \
480 \
481 addr = (void *)((_esmp)->esm_base + (_offset)); \
482 \
483 (_eop)->eo_u64[0] = *addr++; \
484 (_eop)->eo_u64[1] = *addr; \
485 \
486 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
487 uint32_t, (_eop)->eo_u32[3], \
488 uint32_t, (_eop)->eo_u32[2], \
489 uint32_t, (_eop)->eo_u32[1], \
490 uint32_t, (_eop)->eo_u32[0]); \
491 \
492 _NOTE(CONSTANTCONDITION) \
493 } while (B_FALSE)
494 #else
495 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
496 do { \
497 uint32_t *addr; \
498 \
499 _NOTE(CONSTANTCONDITION) \
500 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
501 ("not power of 2 aligned")); \
502 \
503 addr = (void *)((_esmp)->esm_base + (_offset)); \
504 \
505 (_eop)->eo_u32[0] = *addr++; \
506 (_eop)->eo_u32[1] = *addr++; \
507 (_eop)->eo_u32[2] = *addr++; \
508 (_eop)->eo_u32[3] = *addr; \
509 \
510 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
511 uint32_t, (_eop)->eo_u32[3], \
512 uint32_t, (_eop)->eo_u32[2], \
513 uint32_t, (_eop)->eo_u32[1], \
514 uint32_t, (_eop)->eo_u32[0]); \
515 \
516 _NOTE(CONSTANTCONDITION) \
517 } while (B_FALSE)
518 #endif
519
520 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
521 do { \
522 uint32_t *addr; \
523 \
524 _NOTE(CONSTANTCONDITION) \
525 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
526 ("not power of 2 aligned")); \
527 \
528 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
529 uint32_t, (_edp)->ed_u32[0]); \
530 \
531 addr = (void *)((_esmp)->esm_base + (_offset)); \
532 \
533 *addr = (_edp)->ed_u32[0]; \
534 \
535 _NOTE(CONSTANTCONDITION) \
536 } while (B_FALSE)
537
538 #if defined(__x86_64__)
539 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
540 do { \
541 uint64_t *addr; \
542 \
543 _NOTE(CONSTANTCONDITION) \
544 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
545 ("not power of 2 aligned")); \
546 \
547 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
548 uint32_t, (_eqp)->eq_u32[1], \
549 uint32_t, (_eqp)->eq_u32[0]); \
550 \
551 addr = (void *)((_esmp)->esm_base + (_offset)); \
552 \
553 *addr = (_eqp)->eq_u64[0]; \
554 \
555 _NOTE(CONSTANTCONDITION) \
556 } while (B_FALSE)
557
558 #else
559 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
560 do { \
561 uint32_t *addr; \
562 \
563 _NOTE(CONSTANTCONDITION) \
564 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
565 ("not power of 2 aligned")); \
566 \
567 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
568 uint32_t, (_eqp)->eq_u32[1], \
569 uint32_t, (_eqp)->eq_u32[0]); \
570 \
571 addr = (void *)((_esmp)->esm_base + (_offset)); \
572 \
573 *addr++ = (_eqp)->eq_u32[0]; \
574 *addr = (_eqp)->eq_u32[1]; \
575 \
576 _NOTE(CONSTANTCONDITION) \
577 } while (B_FALSE)
578 #endif
579
580 #if defined(__x86_64__)
581 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
582 do { \
583 uint64_t *addr; \
584 \
585 _NOTE(CONSTANTCONDITION) \
586 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
587 ("not power of 2 aligned")); \
588 \
589 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
590 uint32_t, (_eop)->eo_u32[3], \
591 uint32_t, (_eop)->eo_u32[2], \
592 uint32_t, (_eop)->eo_u32[1], \
593 uint32_t, (_eop)->eo_u32[0]); \
594 \
595 addr = (void *)((_esmp)->esm_base + (_offset)); \
596 \
597 *addr++ = (_eop)->eo_u64[0]; \
598 *addr = (_eop)->eo_u64[1]; \
599 \
600 _NOTE(CONSTANTCONDITION) \
601 } while (B_FALSE)
602 #else
603 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
604 do { \
605 uint32_t *addr; \
606 \
607 _NOTE(CONSTANTCONDITION) \
608 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
609 ("not power of 2 aligned")); \
610 \
611 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
612 uint32_t, (_eop)->eo_u32[3], \
613 uint32_t, (_eop)->eo_u32[2], \
614 uint32_t, (_eop)->eo_u32[1], \
615 uint32_t, (_eop)->eo_u32[0]); \
616 \
617 addr = (void *)((_esmp)->esm_base + (_offset)); \
618 \
619 *addr++ = (_eop)->eo_u32[0]; \
620 *addr++ = (_eop)->eo_u32[1]; \
621 *addr++ = (_eop)->eo_u32[2]; \
622 *addr = (_eop)->eo_u32[3]; \
623 \
624 _NOTE(CONSTANTCONDITION) \
625 } while (B_FALSE)
626 #endif
627
628 #define EFSYS_MEM_ADDR(_esmp) \
629 ((_esmp)->esm_addr)
630
631 #define EFSYS_MEM_IS_NULL(_esmp) \
632 ((_esmp)->esm_base == NULL)
633
634 /* BAR */
635
636 #define SFXGE_LOCK_NAME_MAX 16
637
638 typedef struct efsys_bar_s {
639 struct mtx esb_lock;
640 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
641 bus_space_tag_t esb_tag;
642 bus_space_handle_t esb_handle;
643 int esb_rid;
644 struct resource *esb_res;
645 } efsys_bar_t;
646
647 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
648 do { \
649 snprintf((_esbp)->esb_lock_name, \
650 sizeof((_esbp)->esb_lock_name), \
651 "%s:bar", (_ifname)); \
652 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
653 NULL, MTX_DEF); \
654 _NOTE(CONSTANTCONDITION) \
655 } while (B_FALSE)
656 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
657 mtx_destroy(&(_esbp)->esb_lock)
658 #define SFXGE_BAR_LOCK(_esbp) \
659 mtx_lock(&(_esbp)->esb_lock)
660 #define SFXGE_BAR_UNLOCK(_esbp) \
661 mtx_unlock(&(_esbp)->esb_lock)
662
663 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
664 do { \
665 _NOTE(CONSTANTCONDITION) \
666 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
667 ("not power of 2 aligned")); \
668 \
669 _NOTE(CONSTANTCONDITION) \
670 if (_lock) \
671 SFXGE_BAR_LOCK(_esbp); \
672 \
673 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
674 (_esbp)->esb_tag, (_esbp)->esb_handle, \
675 (_offset)); \
676 \
677 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
678 uint32_t, (_edp)->ed_u32[0]); \
679 \
680 _NOTE(CONSTANTCONDITION) \
681 if (_lock) \
682 SFXGE_BAR_UNLOCK(_esbp); \
683 _NOTE(CONSTANTCONDITION) \
684 } while (B_FALSE)
685
686 #if defined(SFXGE_USE_BUS_SPACE_8)
687 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
688 do { \
689 _NOTE(CONSTANTCONDITION) \
690 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
691 ("not power of 2 aligned")); \
692 \
693 SFXGE_BAR_LOCK(_esbp); \
694 \
695 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
696 (_esbp)->esb_tag, (_esbp)->esb_handle, \
697 (_offset)); \
698 \
699 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
700 uint32_t, (_eqp)->eq_u32[1], \
701 uint32_t, (_eqp)->eq_u32[0]); \
702 \
703 SFXGE_BAR_UNLOCK(_esbp); \
704 _NOTE(CONSTANTCONDITION) \
705 } while (B_FALSE)
706
707 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
708 do { \
709 _NOTE(CONSTANTCONDITION) \
710 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
711 ("not power of 2 aligned")); \
712 \
713 _NOTE(CONSTANTCONDITION) \
714 if (_lock) \
715 SFXGE_BAR_LOCK(_esbp); \
716 \
717 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
718 (_esbp)->esb_tag, (_esbp)->esb_handle, \
719 (_offset)); \
720 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
721 (_esbp)->esb_tag, (_esbp)->esb_handle, \
722 (_offset) + 8); \
723 \
724 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
725 uint32_t, (_eop)->eo_u32[3], \
726 uint32_t, (_eop)->eo_u32[2], \
727 uint32_t, (_eop)->eo_u32[1], \
728 uint32_t, (_eop)->eo_u32[0]); \
729 \
730 _NOTE(CONSTANTCONDITION) \
731 if (_lock) \
732 SFXGE_BAR_UNLOCK(_esbp); \
733 _NOTE(CONSTANTCONDITION) \
734 } while (B_FALSE)
735
736 #else
737 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
738 do { \
739 _NOTE(CONSTANTCONDITION) \
740 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
741 ("not power of 2 aligned")); \
742 \
743 SFXGE_BAR_LOCK(_esbp); \
744 \
745 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
746 (_esbp)->esb_tag, (_esbp)->esb_handle, \
747 (_offset)); \
748 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
749 (_esbp)->esb_tag, (_esbp)->esb_handle, \
750 (_offset) + 4); \
751 \
752 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
753 uint32_t, (_eqp)->eq_u32[1], \
754 uint32_t, (_eqp)->eq_u32[0]); \
755 \
756 SFXGE_BAR_UNLOCK(_esbp); \
757 _NOTE(CONSTANTCONDITION) \
758 } while (B_FALSE)
759
760 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
761 do { \
762 _NOTE(CONSTANTCONDITION) \
763 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
764 ("not power of 2 aligned")); \
765 \
766 _NOTE(CONSTANTCONDITION) \
767 if (_lock) \
768 SFXGE_BAR_LOCK(_esbp); \
769 \
770 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
771 (_esbp)->esb_tag, (_esbp)->esb_handle, \
772 (_offset)); \
773 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
774 (_esbp)->esb_tag, (_esbp)->esb_handle, \
775 (_offset) + 4); \
776 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
777 (_esbp)->esb_tag, (_esbp)->esb_handle, \
778 (_offset) + 8); \
779 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
780 (_esbp)->esb_tag, (_esbp)->esb_handle, \
781 (_offset) + 12); \
782 \
783 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
784 uint32_t, (_eop)->eo_u32[3], \
785 uint32_t, (_eop)->eo_u32[2], \
786 uint32_t, (_eop)->eo_u32[1], \
787 uint32_t, (_eop)->eo_u32[0]); \
788 \
789 _NOTE(CONSTANTCONDITION) \
790 if (_lock) \
791 SFXGE_BAR_UNLOCK(_esbp); \
792 _NOTE(CONSTANTCONDITION) \
793 } while (B_FALSE)
794 #endif
795
796 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
797 do { \
798 _NOTE(CONSTANTCONDITION) \
799 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
800 ("not power of 2 aligned")); \
801 \
802 _NOTE(CONSTANTCONDITION) \
803 if (_lock) \
804 SFXGE_BAR_LOCK(_esbp); \
805 \
806 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
807 uint32_t, (_edp)->ed_u32[0]); \
808 \
809 /* \
810 * Make sure that previous writes to the dword have \
811 * been done. It should be cheaper than barrier just \
812 * after the write below. \
813 */ \
814 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
815 (_offset), sizeof (efx_dword_t), \
816 BUS_SPACE_BARRIER_WRITE); \
817 bus_space_write_stream_4((_esbp)->esb_tag, \
818 (_esbp)->esb_handle, \
819 (_offset), (_edp)->ed_u32[0]); \
820 \
821 _NOTE(CONSTANTCONDITION) \
822 if (_lock) \
823 SFXGE_BAR_UNLOCK(_esbp); \
824 _NOTE(CONSTANTCONDITION) \
825 } while (B_FALSE)
826
827 #if defined(SFXGE_USE_BUS_SPACE_8)
828 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
829 do { \
830 _NOTE(CONSTANTCONDITION) \
831 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
832 ("not power of 2 aligned")); \
833 \
834 SFXGE_BAR_LOCK(_esbp); \
835 \
836 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
837 uint32_t, (_eqp)->eq_u32[1], \
838 uint32_t, (_eqp)->eq_u32[0]); \
839 \
840 /* \
841 * Make sure that previous writes to the qword have \
842 * been done. It should be cheaper than barrier just \
843 * after the write below. \
844 */ \
845 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
846 (_offset), sizeof (efx_qword_t), \
847 BUS_SPACE_BARRIER_WRITE); \
848 bus_space_write_stream_8((_esbp)->esb_tag, \
849 (_esbp)->esb_handle, \
850 (_offset), (_eqp)->eq_u64[0]); \
851 \
852 SFXGE_BAR_UNLOCK(_esbp); \
853 _NOTE(CONSTANTCONDITION) \
854 } while (B_FALSE)
855 #else
856 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
857 do { \
858 _NOTE(CONSTANTCONDITION) \
859 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
860 ("not power of 2 aligned")); \
861 \
862 SFXGE_BAR_LOCK(_esbp); \
863 \
864 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
865 uint32_t, (_eqp)->eq_u32[1], \
866 uint32_t, (_eqp)->eq_u32[0]); \
867 \
868 /* \
869 * Make sure that previous writes to the qword have \
870 * been done. It should be cheaper than barrier just \
871 * after the last write below. \
872 */ \
873 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
874 (_offset), sizeof (efx_qword_t), \
875 BUS_SPACE_BARRIER_WRITE); \
876 bus_space_write_stream_4((_esbp)->esb_tag, \
877 (_esbp)->esb_handle, \
878 (_offset), (_eqp)->eq_u32[0]); \
879 /* \
880 * It should be guaranteed that the last dword comes \
881 * the last, so barrier entire qword to be sure that \
882 * neither above nor below writes are reordered. \
883 */ \
884 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
885 (_offset), sizeof (efx_qword_t), \
886 BUS_SPACE_BARRIER_WRITE); \
887 bus_space_write_stream_4((_esbp)->esb_tag, \
888 (_esbp)->esb_handle, \
889 (_offset) + 4, (_eqp)->eq_u32[1]); \
890 \
891 SFXGE_BAR_UNLOCK(_esbp); \
892 _NOTE(CONSTANTCONDITION) \
893 } while (B_FALSE)
894 #endif
895
896 /*
897 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
898 * (required by PIO hardware)
899 */
900 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \
901 do { \
902 _NOTE(CONSTANTCONDITION) \
903 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
904 ("not power of 2 aligned")); \
905 \
906 (void) (_esbp); \
907 \
908 /* FIXME: Perform a 64-bit write */ \
909 KASSERT(0, ("not implemented")); \
910 \
911 _NOTE(CONSTANTCONDITION) \
912 } while (B_FALSE)
913
914 #if defined(SFXGE_USE_BUS_SPACE_8)
915 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
916 do { \
917 _NOTE(CONSTANTCONDITION) \
918 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
919 ("not power of 2 aligned")); \
920 \
921 _NOTE(CONSTANTCONDITION) \
922 if (_lock) \
923 SFXGE_BAR_LOCK(_esbp); \
924 \
925 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
926 uint32_t, (_eop)->eo_u32[3], \
927 uint32_t, (_eop)->eo_u32[2], \
928 uint32_t, (_eop)->eo_u32[1], \
929 uint32_t, (_eop)->eo_u32[0]); \
930 \
931 /* \
932 * Make sure that previous writes to the oword have \
933 * been done. It should be cheaper than barrier just \
934 * after the last write below. \
935 */ \
936 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
937 (_offset), sizeof (efx_oword_t), \
938 BUS_SPACE_BARRIER_WRITE); \
939 bus_space_write_stream_8((_esbp)->esb_tag, \
940 (_esbp)->esb_handle, \
941 (_offset), (_eop)->eo_u64[0]); \
942 /* \
943 * It should be guaranteed that the last qword comes \
944 * the last, so barrier entire oword to be sure that \
945 * neither above nor below writes are reordered. \
946 */ \
947 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
948 (_offset), sizeof (efx_oword_t), \
949 BUS_SPACE_BARRIER_WRITE); \
950 bus_space_write_stream_8((_esbp)->esb_tag, \
951 (_esbp)->esb_handle, \
952 (_offset) + 8, (_eop)->eo_u64[1]); \
953 \
954 _NOTE(CONSTANTCONDITION) \
955 if (_lock) \
956 SFXGE_BAR_UNLOCK(_esbp); \
957 _NOTE(CONSTANTCONDITION) \
958 } while (B_FALSE)
959
960 #else
961 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
962 do { \
963 _NOTE(CONSTANTCONDITION) \
964 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
965 ("not power of 2 aligned")); \
966 \
967 _NOTE(CONSTANTCONDITION) \
968 if (_lock) \
969 SFXGE_BAR_LOCK(_esbp); \
970 \
971 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
972 uint32_t, (_eop)->eo_u32[3], \
973 uint32_t, (_eop)->eo_u32[2], \
974 uint32_t, (_eop)->eo_u32[1], \
975 uint32_t, (_eop)->eo_u32[0]); \
976 \
977 /* \
978 * Make sure that previous writes to the oword have \
979 * been done. It should be cheaper than barrier just \
980 * after the last write below. \
981 */ \
982 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
983 (_offset), sizeof (efx_oword_t), \
984 BUS_SPACE_BARRIER_WRITE); \
985 bus_space_write_stream_4((_esbp)->esb_tag, \
986 (_esbp)->esb_handle, \
987 (_offset), (_eop)->eo_u32[0]); \
988 bus_space_write_stream_4((_esbp)->esb_tag, \
989 (_esbp)->esb_handle, \
990 (_offset) + 4, (_eop)->eo_u32[1]); \
991 bus_space_write_stream_4((_esbp)->esb_tag, \
992 (_esbp)->esb_handle, \
993 (_offset) + 8, (_eop)->eo_u32[2]); \
994 /* \
995 * It should be guaranteed that the last dword comes \
996 * the last, so barrier entire oword to be sure that \
997 * neither above nor below writes are reordered. \
998 */ \
999 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
1000 (_offset), sizeof (efx_oword_t), \
1001 BUS_SPACE_BARRIER_WRITE); \
1002 bus_space_write_stream_4((_esbp)->esb_tag, \
1003 (_esbp)->esb_handle, \
1004 (_offset) + 12, (_eop)->eo_u32[3]); \
1005 \
1006 _NOTE(CONSTANTCONDITION) \
1007 if (_lock) \
1008 SFXGE_BAR_UNLOCK(_esbp); \
1009 _NOTE(CONSTANTCONDITION) \
1010 } while (B_FALSE)
1011 #endif
1012
1013 /* Use the standard octo-word write for doorbell writes */
1014 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \
1015 do { \
1016 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \
1017 _NOTE(CONSTANTCONDITION) \
1018 } while (B_FALSE)
1019
1020 /* SPIN */
1021
1022 #define EFSYS_SPIN(_us) \
1023 do { \
1024 DELAY(_us); \
1025 _NOTE(CONSTANTCONDITION) \
1026 } while (B_FALSE)
1027
1028 #define EFSYS_SLEEP EFSYS_SPIN
1029
1030 /* BARRIERS */
1031
1032 #define EFSYS_MEM_READ_BARRIER() rmb()
1033 #define EFSYS_PIO_WRITE_BARRIER()
1034
1035 /* DMA SYNC */
1036 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \
1037 do { \
1038 bus_dmamap_sync((_esmp)->esm_tag, \
1039 (_esmp)->esm_map, \
1040 BUS_DMASYNC_POSTREAD); \
1041 _NOTE(CONSTANTCONDITION) \
1042 } while (B_FALSE)
1043
1044 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \
1045 do { \
1046 bus_dmamap_sync((_esmp)->esm_tag, \
1047 (_esmp)->esm_map, \
1048 BUS_DMASYNC_PREWRITE); \
1049 _NOTE(CONSTANTCONDITION) \
1050 } while (B_FALSE)
1051
1052 /* TIMESTAMP */
1053
1054 typedef clock_t efsys_timestamp_t;
1055
1056 #define EFSYS_TIMESTAMP(_usp) \
1057 do { \
1058 clock_t now; \
1059 \
1060 now = ticks; \
1061 *(_usp) = now * hz / 1000000; \
1062 _NOTE(CONSTANTCONDITION) \
1063 } while (B_FALSE)
1064
1065 /* KMEM */
1066
1067 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
1068 do { \
1069 (_esip) = (_esip); \
1070 /* \
1071 * The macro is used in non-sleepable contexts, for \
1072 * example, holding a mutex. \
1073 */ \
1074 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
1075 _NOTE(CONSTANTCONDITION) \
1076 } while (B_FALSE)
1077
1078 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
1079 do { \
1080 (void) (_esip); \
1081 (void) (_size); \
1082 free((_p), M_SFXGE); \
1083 _NOTE(CONSTANTCONDITION) \
1084 } while (B_FALSE)
1085
1086 /* LOCK */
1087
1088 typedef struct efsys_lock_s {
1089 struct mtx lock;
1090 char lock_name[SFXGE_LOCK_NAME_MAX];
1091 } efsys_lock_t;
1092
1093 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
1094 do { \
1095 efsys_lock_t *__eslp = (_eslp); \
1096 \
1097 snprintf((__eslp)->lock_name, \
1098 sizeof((__eslp)->lock_name), \
1099 "%s:%s", (_ifname), (_label)); \
1100 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
1101 NULL, MTX_DEF); \
1102 } while (B_FALSE)
1103 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
1104 mtx_destroy(&(_eslp)->lock)
1105 #define SFXGE_EFSYS_LOCK(_eslp) \
1106 mtx_lock(&(_eslp)->lock)
1107 #define SFXGE_EFSYS_UNLOCK(_eslp) \
1108 mtx_unlock(&(_eslp)->lock)
1109 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
1110 mtx_assert(&(_eslp)->lock, MA_OWNED)
1111
1112 #define EFSYS_LOCK_MAGIC 0x000010c4
1113
1114 #define EFSYS_LOCK(_lockp, _state) \
1115 do { \
1116 SFXGE_EFSYS_LOCK(_lockp); \
1117 (_state) = EFSYS_LOCK_MAGIC; \
1118 _NOTE(CONSTANTCONDITION) \
1119 } while (B_FALSE)
1120
1121 #define EFSYS_UNLOCK(_lockp, _state) \
1122 do { \
1123 if ((_state) != EFSYS_LOCK_MAGIC) \
1124 KASSERT(B_FALSE, ("not locked")); \
1125 SFXGE_EFSYS_UNLOCK(_lockp); \
1126 _NOTE(CONSTANTCONDITION) \
1127 } while (B_FALSE)
1128
1129 /* PREEMPT */
1130
1131 #define EFSYS_PREEMPT_DISABLE(_state) \
1132 do { \
1133 (_state) = (_state); \
1134 critical_enter(); \
1135 _NOTE(CONSTANTCONDITION) \
1136 } while (B_FALSE)
1137
1138 #define EFSYS_PREEMPT_ENABLE(_state) \
1139 do { \
1140 (_state) = (_state); \
1141 critical_exit(_state); \
1142 _NOTE(CONSTANTCONDITION) \
1143 } while (B_FALSE)
1144
1145 /* STAT */
1146
1147 typedef uint64_t efsys_stat_t;
1148
1149 #define EFSYS_STAT_INCR(_knp, _delta) \
1150 do { \
1151 *(_knp) += (_delta); \
1152 _NOTE(CONSTANTCONDITION) \
1153 } while (B_FALSE)
1154
1155 #define EFSYS_STAT_DECR(_knp, _delta) \
1156 do { \
1157 *(_knp) -= (_delta); \
1158 _NOTE(CONSTANTCONDITION) \
1159 } while (B_FALSE)
1160
1161 #define EFSYS_STAT_SET(_knp, _val) \
1162 do { \
1163 *(_knp) = (_val); \
1164 _NOTE(CONSTANTCONDITION) \
1165 } while (B_FALSE)
1166
1167 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1168 do { \
1169 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1170 _NOTE(CONSTANTCONDITION) \
1171 } while (B_FALSE)
1172
1173 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1174 do { \
1175 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1176 _NOTE(CONSTANTCONDITION) \
1177 } while (B_FALSE)
1178
1179 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1180 do { \
1181 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1182 _NOTE(CONSTANTCONDITION) \
1183 } while (B_FALSE)
1184
1185 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1186 do { \
1187 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1188 _NOTE(CONSTANTCONDITION) \
1189 } while (B_FALSE)
1190
1191 /* ERR */
1192
1193 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1194 uint32_t, uint32_t);
1195
1196 #if EFSYS_OPT_DECODE_INTR_FATAL
1197 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1198 do { \
1199 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1200 _NOTE(CONSTANTCONDITION) \
1201 } while (B_FALSE)
1202 #endif
1203
1204 /* ASSERT */
1205
1206 #define EFSYS_ASSERT(_exp) do { \
1207 if (!(_exp)) \
1208 panic("%s", #_exp); \
1209 } while (0)
1210
1211 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1212 const _t __x = (_t)(_x); \
1213 const _t __y = (_t)(_y); \
1214 if (!(__x _op __y)) \
1215 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1216 } while(0)
1217
1218 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1219 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1220 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1221
1222 /* ROTATE */
1223
1224 #define EFSYS_HAS_ROTL_DWORD 0
1225
1226 #ifdef __cplusplus
1227 }
1228 #endif
1229
1230 #endif /* _SYS_EFSYS_H */
1231