1 /*
2 * Copyright (c) 2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 */
29
30 #ifndef __BNXT_RE_MAIN_H__
31 #define __BNXT_RE_MAIN_H__
32
33 #include <sys/param.h>
34 #include <sys/queue.h>
35
36 #include <infiniband/driver.h>
37 #include <infiniband/endian.h>
38 #include <infiniband/udma_barrier.h>
39
40 #include <inttypes.h>
41 #include <math.h>
42 #include <pthread.h>
43 #include <stdatomic.h>
44 #include <stdbool.h>
45 #include <stddef.h>
46 #include <stdio.h>
47 #include <stdlib.h>
48
49 #include "abi.h"
50 #include "list.h"
51 #include "memory.h"
52
53 #define DEV "bnxt_re : "
54 #define BNXT_RE_UD_QP_STALL 0x400000
55
56 #define CHIP_NUM_57508 0x1750
57 #define CHIP_NUM_57504 0x1751
58 #define CHIP_NUM_57502 0x1752
59 #define CHIP_NUM_58818 0xd818
60 #define CHIP_NUM_57608 0x1760
61
62 #define BNXT_NSEC_PER_SEC 1000000000UL
63
64 struct bnxt_re_chip_ctx {
65 __u16 chip_num;
66 __u8 chip_rev;
67 __u8 chip_metal;
68 bool chip_is_gen_p5_thor2;
69 };
70
71 #define BNXT_RE_MAP_WC 0x1000
72 #define BNXT_RE_DBR_PAGE 0x2000
73 #define BNXT_RE_DB_RECOVERY_PAGE 0x3000
74
75 #define BNXT_RE_DB_REPLAY_YIELD_CNT 256
76 #define BNXT_RE_DB_KEY_INVALID -1
77 #define BNXT_RE_MAX_DO_PACING 0xFFFF
78 #define bnxt_re_wm_barrier() udma_to_device_barrier()
79 #define unlikely(x) __builtin_expect(!!(x), 0)
80 #define likely(x) __builtin_expect(!!(x), 1)
81
82 #define CNA(v, d) \
83 { .vendor = PCI_VENDOR_ID_##v, \
84 .device = d }
85 #define BNXT_RE_DEFINE_CNA_TABLE(_name) \
86 static const struct { \
87 unsigned vendor; \
88 unsigned device; \
89 } _name[]
90
91 struct bnxt_re_dpi {
92 __u32 dpindx;
93 __u32 wcdpi;
94 __u64 *dbpage;
95 __u64 *wcdbpg;
96 };
97
98 struct bnxt_re_pd {
99 struct ibv_pd ibvpd;
100 uint32_t pdid;
101 };
102
103 struct xorshift32_state {
104 uint32_t seed;
105 };
106
107 struct bnxt_re_cq {
108 struct ibv_cq ibvcq;
109 struct bnxt_re_list_head sfhead;
110 struct bnxt_re_list_head rfhead;
111 struct bnxt_re_list_head prev_cq_head;
112 struct bnxt_re_context *cntx;
113 struct bnxt_re_queue *cqq;
114 struct bnxt_re_dpi *udpi;
115 struct bnxt_re_mem *resize_mem;
116 struct bnxt_re_mem *mem;
117 struct bnxt_re_list_node dbnode;
118 uint64_t shadow_db_key;
119 uint32_t cqe_sz;
120 uint32_t cqid;
121 struct xorshift32_state rand;
122 int deferred_arm_flags;
123 bool first_arm;
124 bool deferred_arm;
125 bool phase;
126 uint8_t dbr_lock;
127 void *cq_page;
128 };
129
130 struct bnxt_re_push_buffer {
131 __u64 *pbuf; /*push wc buffer */
132 __u64 *wqe; /* hwqe addresses */
133 __u64 *ucdb;
134 uint32_t st_idx;
135 uint32_t qpid;
136 uint16_t wcdpi;
137 uint16_t nbit;
138 uint32_t tail;
139 };
140
141 enum bnxt_re_push_info_mask {
142 BNXT_RE_PUSH_SIZE_MASK = 0x1FUL,
143 BNXT_RE_PUSH_SIZE_SHIFT = 0x18UL
144 };
145
146 struct bnxt_re_db_ppp_hdr {
147 struct bnxt_re_db_hdr db_hdr;
148 __u64 rsv_psz_pidx;
149 };
150
151 struct bnxt_re_push_rec {
152 struct bnxt_re_dpi *udpi;
153 struct bnxt_re_push_buffer *pbuf;
154 __u32 pbmap; /* only 16 bits in use */
155 };
156
157 struct bnxt_re_wrid {
158 uint64_t wrid;
159 int next_idx;
160 uint32_t bytes;
161 uint8_t sig;
162 uint8_t slots;
163 uint8_t wc_opcd;
164 };
165
166 struct bnxt_re_qpcap {
167 uint32_t max_swr;
168 uint32_t max_rwr;
169 uint32_t max_ssge;
170 uint32_t max_rsge;
171 uint32_t max_inline;
172 uint8_t sqsig;
173 uint8_t is_atomic_cap;
174 };
175
176 struct bnxt_re_srq {
177 struct ibv_srq ibvsrq;
178 struct ibv_srq_attr cap;
179 uint32_t srqid;
180 struct bnxt_re_context *uctx;
181 struct bnxt_re_queue *srqq;
182 struct bnxt_re_wrid *srwrid;
183 struct bnxt_re_dpi *udpi;
184 struct bnxt_re_mem *mem;
185 int start_idx;
186 int last_idx;
187 struct bnxt_re_list_node dbnode;
188 uint64_t shadow_db_key;
189 struct xorshift32_state rand;
190 uint8_t dbr_lock;
191 bool arm_req;
192 };
193
194 struct bnxt_re_joint_queue {
195 struct bnxt_re_context *cntx;
196 struct bnxt_re_queue *hwque;
197 struct bnxt_re_wrid *swque;
198 uint32_t start_idx;
199 uint32_t last_idx;
200 };
201
202 struct bnxt_re_qp {
203 struct ibv_qp ibvqp;
204 struct bnxt_re_qpcap cap;
205 struct bnxt_re_context *cntx;
206 struct bnxt_re_chip_ctx *cctx;
207 struct bnxt_re_joint_queue *jsqq;
208 struct bnxt_re_joint_queue *jrqq;
209 struct bnxt_re_dpi *udpi;
210 uint64_t wqe_cnt;
211 uint16_t mtu;
212 uint16_t qpst;
213 uint8_t qptyp;
214 uint8_t qpmode;
215 uint8_t push_st_en;
216 uint8_t ppp_idx;
217 uint32_t sq_psn;
218 uint32_t sq_msn;
219 uint32_t qpid;
220 uint16_t max_push_sz;
221 uint8_t sq_dbr_lock;
222 uint8_t rq_dbr_lock;
223 struct xorshift32_state rand;
224 struct bnxt_re_list_node snode;
225 struct bnxt_re_list_node rnode;
226 struct bnxt_re_srq *srq;
227 struct bnxt_re_cq *rcq;
228 struct bnxt_re_cq *scq;
229 struct bnxt_re_mem *mem;/* at cl 6 */
230 struct bnxt_re_list_node dbnode;
231 uint64_t sq_shadow_db_key;
232 uint64_t rq_shadow_db_key;
233 };
234
235 struct bnxt_re_mr {
236 struct ibv_mr vmr;
237 };
238
239 struct bnxt_re_ah {
240 struct ibv_ah ibvah;
241 struct bnxt_re_pd *pd;
242 uint32_t avid;
243 };
244
245 struct bnxt_re_dev {
246 struct verbs_device vdev;
247 struct ibv_device_attr devattr;
248 uint32_t pg_size;
249 uint32_t cqe_size;
250 uint32_t max_cq_depth;
251 uint8_t abi_version;
252 };
253
254 struct bnxt_re_res_list {
255 struct bnxt_re_list_head head;
256 pthread_spinlock_t lock;
257 };
258
259 struct bnxt_re_context {
260 struct ibv_context ibvctx;
261 struct bnxt_re_dev *rdev;
262 struct bnxt_re_chip_ctx *cctx;
263 uint64_t comp_mask;
264 struct bnxt_re_dpi udpi;
265 uint32_t dev_id;
266 uint32_t max_qp;
267 uint32_t max_srq;
268 uint32_t modes;
269 void *shpg;
270 pthread_mutex_t shlock;
271 struct bnxt_re_push_rec *pbrec;
272 void *dbr_page;
273 void *bar_map;
274 struct bnxt_re_res_list qp_dbr_res;
275 struct bnxt_re_res_list cq_dbr_res;
276 struct bnxt_re_res_list srq_dbr_res;
277 void *db_recovery_page;
278 struct ibv_comp_channel *dbr_ev_chan;
279 struct ibv_cq *dbr_cq;
280 pthread_t dbr_thread;
281 uint64_t replay_cnt;
282 };
283
284 struct bnxt_re_pacing_data {
285 uint32_t do_pacing;
286 uint32_t pacing_th;
287 uint32_t dev_err_state;
288 uint32_t alarm_th;
289 };
290
291 /* Chip context related functions */
292 bool _is_chip_gen_p5(struct bnxt_re_chip_ctx *cctx);
293 bool _is_chip_a0(struct bnxt_re_chip_ctx *cctx);
294 bool _is_chip_thor2(struct bnxt_re_chip_ctx *cctx);
295 bool _is_chip_gen_p5_thor2(struct bnxt_re_chip_ctx *cctx);
296
297 /* DB ring functions used internally*/
298 void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp);
299 void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp);
300 void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq);
301 void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq);
302 void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq);
303 void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag);
304
305 void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp,
306 struct bnxt_re_push_buffer *pbuf);
307 void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp,
308 struct bnxt_re_push_buffer *pbuf);
309 void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp,
310 struct bnxt_re_push_buffer *pbuf,
311 uint32_t idx);
312
313 void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf,
314 struct bnxt_re_qp *qp, uint8_t len, uint32_t idx);
315 int bnxt_re_init_pbuf_list(struct bnxt_re_context *cntx);
316 void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx);
317 struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en,
318 uint8_t ppp_idx,
319 struct bnxt_re_context *cntx);
320 void bnxt_re_put_pbuf(struct bnxt_re_context *cntx,
321 struct bnxt_re_push_buffer *pbuf);
322
323 void bnxt_re_db_recovery(struct bnxt_re_context *cntx);
324 void *bnxt_re_dbr_thread(void *arg);
325 bool _is_db_drop_recovery_enable(struct bnxt_re_context *cntx);
326 int bnxt_re_poll_kernel_cq(struct bnxt_re_cq *cq);
327 extern int bnxt_single_threaded;
328 extern int bnxt_dyn_debug;
329
330 #define bnxt_re_trace(fmt, ...) \
331 { \
332 if (bnxt_dyn_debug) \
333 fprintf(stderr, fmt, ##__VA_ARGS__); \
334 }
335
336 /* pointer conversion functions*/
to_bnxt_re_dev(struct ibv_device * ibvdev)337 static inline struct bnxt_re_dev *to_bnxt_re_dev(struct ibv_device *ibvdev)
338 {
339 return container_of(ibvdev, struct bnxt_re_dev, vdev);
340 }
341
to_bnxt_re_context(struct ibv_context * ibvctx)342 static inline struct bnxt_re_context *to_bnxt_re_context(
343 struct ibv_context *ibvctx)
344 {
345 return container_of(ibvctx, struct bnxt_re_context, ibvctx);
346 }
347
to_bnxt_re_pd(struct ibv_pd * ibvpd)348 static inline struct bnxt_re_pd *to_bnxt_re_pd(struct ibv_pd *ibvpd)
349 {
350 return container_of(ibvpd, struct bnxt_re_pd, ibvpd);
351 }
352
to_bnxt_re_cq(struct ibv_cq * ibvcq)353 static inline struct bnxt_re_cq *to_bnxt_re_cq(struct ibv_cq *ibvcq)
354 {
355 return container_of(ibvcq, struct bnxt_re_cq, ibvcq);
356 }
357
to_bnxt_re_qp(struct ibv_qp * ibvqp)358 static inline struct bnxt_re_qp *to_bnxt_re_qp(struct ibv_qp *ibvqp)
359 {
360 return container_of(ibvqp, struct bnxt_re_qp, ibvqp);
361 }
362
to_bnxt_re_srq(struct ibv_srq * ibvsrq)363 static inline struct bnxt_re_srq *to_bnxt_re_srq(struct ibv_srq *ibvsrq)
364 {
365 return container_of(ibvsrq, struct bnxt_re_srq, ibvsrq);
366 }
367
to_bnxt_re_ah(struct ibv_ah * ibvah)368 static inline struct bnxt_re_ah *to_bnxt_re_ah(struct ibv_ah *ibvah)
369 {
370 return container_of(ibvah, struct bnxt_re_ah, ibvah);
371 }
372
373 /* CQE manipulations */
374 #define bnxt_re_get_cqe_sz() (sizeof(struct bnxt_re_req_cqe) + \
375 sizeof(struct bnxt_re_bcqe))
376 #define bnxt_re_get_sqe_hdr_sz() (sizeof(struct bnxt_re_bsqe) + \
377 sizeof(struct bnxt_re_send))
378 #define bnxt_re_get_srqe_hdr_sz() (sizeof(struct bnxt_re_brqe) + \
379 sizeof(struct bnxt_re_srqe))
380 #define bnxt_re_get_srqe_sz() (sizeof(struct bnxt_re_brqe) + \
381 sizeof(struct bnxt_re_srqe) + \
382 BNXT_RE_MAX_INLINE_SIZE)
383 #define bnxt_re_is_cqe_valid(valid, phase) \
384 (((valid) & BNXT_RE_BCQE_PH_MASK) == (phase))
385
bnxt_re_change_cq_phase(struct bnxt_re_cq * cq)386 static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq)
387 {
388 if (!cq->cqq->head)
389 cq->phase = !(cq->phase & BNXT_RE_BCQE_PH_MASK);
390 }
391
bnxt_re_get_swqe(struct bnxt_re_joint_queue * jqq,uint32_t * wqe_idx)392 static inline void *bnxt_re_get_swqe(struct bnxt_re_joint_queue *jqq,
393 uint32_t *wqe_idx)
394 {
395 if (wqe_idx)
396 *wqe_idx = jqq->start_idx;
397 return &jqq->swque[jqq->start_idx];
398 }
399
bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue * jqq,uint32_t idx)400 static inline void bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue *jqq,
401 uint32_t idx)
402 {
403 jqq->start_idx = jqq->swque[idx].next_idx;
404 }
405
bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue * jqq,uint32_t idx)406 static inline void bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue *jqq,
407 uint32_t idx)
408 {
409 jqq->last_idx = jqq->swque[idx].next_idx;
410 }
411
bnxt_re_init_depth(uint32_t ent,uint64_t cmask)412 static inline uint32_t bnxt_re_init_depth(uint32_t ent, uint64_t cmask)
413 {
414 return cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED ?
415 ent : roundup_pow_of_two(ent);
416 }
417
bnxt_re_get_diff(uint64_t cmask)418 static inline uint32_t bnxt_re_get_diff(uint64_t cmask)
419 {
420 return cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED ?
421 0 : BNXT_RE_FULL_FLAG_DELTA;
422 }
423
bnxt_re_calc_wqe_sz(int nsge)424 static inline int bnxt_re_calc_wqe_sz(int nsge)
425 {
426 /* This is used for both sq and rq. In case hdr size differs
427 * in future move to individual functions.
428 */
429 return sizeof(struct bnxt_re_sge) * nsge + bnxt_re_get_sqe_hdr_sz();
430 }
431
432 /* Helper function to copy to push buffers */
bnxt_re_copy_data_to_pb(struct bnxt_re_push_buffer * pbuf,uint8_t offset,uint32_t idx)433 static inline void bnxt_re_copy_data_to_pb(struct bnxt_re_push_buffer *pbuf,
434 uint8_t offset, uint32_t idx)
435 {
436 __u64 *src;
437 __u64 *dst;
438 int indx;
439
440 for (indx = 0; indx < idx; indx++) {
441 dst = (__u64 *)(pbuf->pbuf + 2 * (indx + offset));
442 src = (__u64 *)pbuf->wqe[indx];
443 iowrite64(dst, *src);
444
445 dst++;
446 src++;
447 iowrite64(dst, *src);
448 }
449 }
450
bnxt_re_dp_spin_init(struct bnxt_spinlock * lock,int pshared,int need_lock)451 static inline int bnxt_re_dp_spin_init(struct bnxt_spinlock *lock, int pshared, int need_lock)
452 {
453 lock->in_use = 0;
454 lock->need_lock = need_lock;
455 return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE);
456 }
457
bnxt_re_dp_spin_destroy(struct bnxt_spinlock * lock)458 static inline int bnxt_re_dp_spin_destroy(struct bnxt_spinlock *lock)
459 {
460 return pthread_spin_destroy(&lock->lock);
461 }
462
bnxt_spin_lock(struct bnxt_spinlock * lock)463 static inline int bnxt_spin_lock(struct bnxt_spinlock *lock)
464 {
465 if (lock->need_lock)
466 return pthread_spin_lock(&lock->lock);
467
468 if (unlikely(lock->in_use)) {
469 fprintf(stderr, "*** ERROR: multithreading violation ***\n"
470 "You are running a multithreaded application but\n"
471 "you set BNXT_SINGLE_THREADED=1. Please unset it.\n");
472 abort();
473 } else {
474 lock->in_use = 1;
475 /* This fence is not at all correct, but it increases the */
476 /* chance that in_use is detected by another thread without */
477 /* much runtime cost. */
478 atomic_thread_fence(memory_order_acq_rel);
479 }
480
481 return 0;
482 }
483
bnxt_spin_unlock(struct bnxt_spinlock * lock)484 static inline int bnxt_spin_unlock(struct bnxt_spinlock *lock)
485 {
486 if (lock->need_lock)
487 return pthread_spin_unlock(&lock->lock);
488
489 lock->in_use = 0;
490 return 0;
491 }
492
timespec_sub(const struct timespec * a,const struct timespec * b,struct timespec * res)493 static void timespec_sub(const struct timespec *a, const struct timespec *b,
494 struct timespec *res)
495 {
496 res->tv_sec = a->tv_sec - b->tv_sec;
497 res->tv_nsec = a->tv_nsec - b->tv_nsec;
498 if (res->tv_nsec < 0) {
499 res->tv_sec--;
500 res->tv_nsec += BNXT_NSEC_PER_SEC;
501 }
502 }
503
504 /*
505 * Function waits in a busy loop for a given nano seconds
506 * The maximum wait period allowed is less than one second
507 */
bnxt_re_sub_sec_busy_wait(uint32_t nsec)508 static inline void bnxt_re_sub_sec_busy_wait(uint32_t nsec)
509 {
510 struct timespec start, cur, res;
511
512 if (nsec >= BNXT_NSEC_PER_SEC)
513 return;
514
515 if (clock_gettime(CLOCK_REALTIME, &start)) {
516 fprintf(stderr, "%s: failed to get time : %d",
517 __func__, errno);
518 return;
519 }
520
521 while (1) {
522 if (clock_gettime(CLOCK_REALTIME, &cur)) {
523 fprintf(stderr, "%s: failed to get time : %d",
524 __func__, errno);
525 return;
526 }
527
528 timespec_sub(&cur, &start, &res);
529 if (res.tv_nsec >= nsec)
530 break;
531 }
532 }
533
534 #define BNXT_RE_HW_RETX(a) ((a)->comp_mask & BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED)
535 #define bnxt_re_dp_spin_lock(lock) bnxt_spin_lock(lock)
536 #define bnxt_re_dp_spin_unlock(lock) bnxt_spin_unlock(lock)
537
538 #endif
539