1 /*
2 * Copyright (c) 2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: Doorbell handling functions.
29 */
30
31 #include <malloc.h>
32 #include <unistd.h>
33
34 #include "abi.h"
35 #include "main.h"
36
37 #define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000
38 #define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00
39
40 #define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000
41 #define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000
42
43 #define BNXT_RE_DB_FIFO_ROOM_SHIFT 15
44 #define BNXT_RE_DB_THRESHOLD 20
45
46 #define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \
47 (_is_chip_thor2((ctx)) ? \
48 BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\
49 BNXT_RE_DB_FIFO_ROOM_MASK_P5)
50 #define BNXT_RE_MAX_FIFO_DEPTH(ctx) \
51 (_is_chip_thor2((ctx)) ? \
52 BNXT_RE_MAX_FIFO_DEPTH_P7 :\
53 BNXT_RE_MAX_FIFO_DEPTH_P5)
54
xorshift32(struct xorshift32_state * state)55 static uint32_t xorshift32(struct xorshift32_state *state)
56 {
57 /* Algorithm "xor" from p. 4 of Marsaglia, "Xorshift RNGs" */
58 uint32_t x = state->seed;
59
60 x ^= x << 13;
61 x ^= x >> 17;
62 x ^= x << 5;
63 return state->seed = x;
64 }
65
rnd(struct xorshift32_state * state,uint16_t range)66 static uint16_t rnd(struct xorshift32_state *state, uint16_t range)
67 {
68 /* range must be a power of 2 - 1 */
69 return (xorshift32(state) & range);
70 }
71
calculate_fifo_occupancy(struct bnxt_re_context * cntx)72 static int calculate_fifo_occupancy(struct bnxt_re_context *cntx)
73 {
74 uint32_t *dbr_map = cntx->bar_map + 0x1a8;
75 uint32_t read_val, fifo_occup;
76
77 read_val = *dbr_map;
78 fifo_occup = BNXT_RE_MAX_FIFO_DEPTH(cntx->cctx) -
79 ((read_val & BNXT_RE_DB_FIFO_ROOM_MASK(cntx->cctx)) >>
80 BNXT_RE_DB_FIFO_ROOM_SHIFT);
81
82 return fifo_occup;
83 }
84
find_min(uint32_t x,uint32_t y)85 static inline uint32_t find_min(uint32_t x, uint32_t y)
86 {
87 return (y > x ? x : y);
88 }
89
bnxt_re_do_pacing(struct bnxt_re_context * cntx,struct xorshift32_state * state)90 int bnxt_re_do_pacing(struct bnxt_re_context *cntx, struct xorshift32_state *state)
91 {
92 /* First 4 bytes of shared page (pacing_info) contains the DBR
93 * pacing information. Second 4 bytes (pacing_th) contains
94 * the pacing threshold value to determine whether to
95 * add delay or not
96 */
97 struct bnxt_re_pacing_data *pacing_data =
98 (struct bnxt_re_pacing_data *)cntx->dbr_page;
99 uint32_t wait_time = 1;
100 uint32_t fifo_occup;
101
102 if (!pacing_data)
103 return 0;
104 /* If the device in error recovery state, return error to
105 * not to ring new doorbells in this state.
106 */
107 if (pacing_data->dev_err_state)
108 return -EFAULT;
109
110 if (rnd(state, BNXT_RE_MAX_DO_PACING) < pacing_data->do_pacing) {
111 while ((fifo_occup = calculate_fifo_occupancy(cntx))
112 > pacing_data->pacing_th) {
113 struct bnxt_re_cq *cq;
114 uint32_t usec_wait;
115
116 if (pacing_data->alarm_th && fifo_occup > pacing_data->alarm_th) {
117 cq = container_of(cntx->dbr_cq, struct bnxt_re_cq, ibvcq);
118 bnxt_re_poll_kernel_cq(cq);
119 }
120 usec_wait = rnd(state, wait_time - 1);
121 if (usec_wait)
122 bnxt_re_sub_sec_busy_wait(usec_wait * 1000);
123 /* wait time capped at 128 us */
124 wait_time = find_min(wait_time * 2, 128);
125 }
126 }
127 return 0;
128 }
129
bnxt_re_ring_db(struct bnxt_re_dpi * dpi,__u64 key,uint64_t * db_key,uint8_t * lock)130 static inline void bnxt_re_ring_db(struct bnxt_re_dpi *dpi, __u64 key,
131 uint64_t *db_key, uint8_t *lock)
132 {
133 while (1) {
134 if (__sync_bool_compare_and_swap(lock, 0, 1)) {
135 *db_key = key;
136 bnxt_re_wm_barrier();
137 iowrite64(dpi->dbpage, key);
138 bnxt_re_wm_barrier();
139 *lock = 0;
140 break;
141 }
142 }
143 }
144
bnxt_re_init_push_hdr(struct bnxt_re_db_hdr * hdr,uint32_t indx,uint32_t qid,uint32_t typ,uint32_t pidx)145 static inline void bnxt_re_init_push_hdr(struct bnxt_re_db_hdr *hdr,
146 uint32_t indx, uint32_t qid,
147 uint32_t typ, uint32_t pidx)
148 {
149 __u64 key_lo, key_hi;
150
151 key_lo = (((pidx & BNXT_RE_DB_PILO_MASK) << BNXT_RE_DB_PILO_SHIFT) |
152 (indx & BNXT_RE_DB_INDX_MASK));
153 key_hi = ((((pidx & BNXT_RE_DB_PIHI_MASK) << BNXT_RE_DB_PIHI_SHIFT) |
154 (qid & BNXT_RE_DB_QID_MASK)) |
155 ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) |
156 (0x1UL << BNXT_RE_DB_VALID_SHIFT));
157 hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32)));
158 }
159
bnxt_re_init_db_hdr(struct bnxt_re_db_hdr * hdr,uint32_t indx,uint32_t toggle,uint32_t qid,uint32_t typ)160 static inline void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr,
161 uint32_t indx, uint32_t toggle,
162 uint32_t qid, uint32_t typ)
163 {
164 __u64 key_lo, key_hi;
165
166 key_lo = htole32(indx | toggle);
167 key_hi = ((qid & BNXT_RE_DB_QID_MASK) |
168 ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) |
169 (0x1UL << BNXT_RE_DB_VALID_SHIFT));
170 hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32)));
171 }
172
__bnxt_re_ring_pend_db(__u64 * ucdb,__u64 key,struct bnxt_re_qp * qp)173 static inline void __bnxt_re_ring_pend_db(__u64 *ucdb, __u64 key,
174 struct bnxt_re_qp *qp)
175 {
176 struct bnxt_re_db_hdr hdr;
177
178 bnxt_re_init_db_hdr(&hdr,
179 (*qp->jsqq->hwque->dbtail |
180 ((qp->jsqq->hwque->flags &
181 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
182 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
183 qp->qpid,
184 BNXT_RE_QUE_TYPE_SQ);
185
186 while (1) {
187 if (__sync_bool_compare_and_swap(&qp->sq_dbr_lock, 0, 1)) {
188 qp->sq_shadow_db_key = hdr.typ_qid_indx;
189 bnxt_re_wm_barrier();
190 iowrite64(ucdb, key);
191 bnxt_re_wm_barrier();
192 qp->sq_dbr_lock = 0;
193 break;
194 }
195 }
196 }
197
bnxt_re_ring_rq_db(struct bnxt_re_qp * qp)198 void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp)
199 {
200 struct bnxt_re_db_hdr hdr;
201
202 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
203 return;
204 bnxt_re_init_db_hdr(&hdr,
205 (*qp->jrqq->hwque->dbtail |
206 ((qp->jrqq->hwque->flags &
207 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
208 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
209 qp->qpid,
210 BNXT_RE_QUE_TYPE_RQ);
211 bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->rq_shadow_db_key,
212 &qp->rq_dbr_lock);
213 }
214
bnxt_re_ring_sq_db(struct bnxt_re_qp * qp)215 void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp)
216 {
217 struct bnxt_re_db_hdr hdr;
218
219 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
220 return;
221 bnxt_re_init_db_hdr(&hdr,
222 (*qp->jsqq->hwque->dbtail |
223 ((qp->jsqq->hwque->flags &
224 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
225 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
226 qp->qpid,
227 BNXT_RE_QUE_TYPE_SQ);
228 bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->sq_shadow_db_key,
229 &qp->sq_dbr_lock);
230 }
231
bnxt_re_ring_srq_db(struct bnxt_re_srq * srq)232 void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq)
233 {
234 struct bnxt_re_db_hdr hdr;
235
236 if (bnxt_re_do_pacing(srq->uctx, &srq->rand))
237 return;
238 bnxt_re_init_db_hdr(&hdr,
239 (srq->srqq->tail |
240 ((srq->srqq->flags &
241 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
242 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
243 srq->srqid, BNXT_RE_QUE_TYPE_SRQ);
244 bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key,
245 &srq->dbr_lock);
246 }
247
bnxt_re_ring_srq_arm(struct bnxt_re_srq * srq)248 void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq)
249 {
250 struct bnxt_re_db_hdr hdr;
251
252 if (bnxt_re_do_pacing(srq->uctx, &srq->rand))
253 return;
254 bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit, 0, srq->srqid,
255 BNXT_RE_QUE_TYPE_SRQ_ARM);
256 bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key,
257 &srq->dbr_lock);
258 }
259
bnxt_re_ring_cq_db(struct bnxt_re_cq * cq)260 void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq)
261 {
262 struct bnxt_re_db_hdr hdr;
263
264 if (bnxt_re_do_pacing(cq->cntx, &cq->rand))
265 return;
266 bnxt_re_init_db_hdr(&hdr,
267 (cq->cqq->head |
268 ((cq->cqq->flags &
269 BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<
270 BNXT_RE_DB_EPOCH_HEAD_SHIFT)), 0,
271 cq->cqid,
272 BNXT_RE_QUE_TYPE_CQ);
273 bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key,
274 &cq->dbr_lock);
275 }
276
bnxt_re_ring_cq_arm_db(struct bnxt_re_cq * cq,uint8_t aflag)277 void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag)
278 {
279 uint32_t *cq_page = cq->cq_page;
280 struct bnxt_re_db_hdr hdr;
281 uint32_t toggle = 0;
282
283 if (cq_page)
284 toggle = *cq_page;
285
286 if (bnxt_re_do_pacing(cq->cntx, &cq->rand))
287 return;
288 bnxt_re_init_db_hdr(&hdr,
289 (cq->cqq->head |
290 ((cq->cqq->flags &
291 BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<
292 BNXT_RE_DB_EPOCH_HEAD_SHIFT)),
293 toggle << BNXT_RE_DB_TOGGLE_SHIFT,
294 cq->cqid, aflag);
295 bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key,
296 &cq->dbr_lock);
297 }
298
bnxt_re_ring_pstart_db(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf)299 void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp,
300 struct bnxt_re_push_buffer *pbuf)
301 {
302 __u64 key;
303
304 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
305 return;
306 key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) <<
307 BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) |
308 ((BNXT_RE_PUSH_TYPE_START & BNXT_RE_DB_TYP_MASK) <<
309 BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT));
310 key <<= 32;
311 key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) <<
312 BNXT_RE_DB_PILO_SHIFT) | (pbuf->st_idx &
313 BNXT_RE_DB_INDX_MASK));
314 bnxt_re_wm_barrier();
315 iowrite64(pbuf->ucdb, key);
316 }
317
bnxt_re_ring_pend_db(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf)318 void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp,
319 struct bnxt_re_push_buffer *pbuf)
320 {
321 __u64 key;
322
323 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
324 return;
325 key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) <<
326 BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) |
327 ((BNXT_RE_PUSH_TYPE_END & BNXT_RE_DB_TYP_MASK) <<
328 BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT));
329 key <<= 32;
330 key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) <<
331 BNXT_RE_DB_PILO_SHIFT) | (pbuf->tail &
332 BNXT_RE_DB_INDX_MASK));
333 __bnxt_re_ring_pend_db(pbuf->ucdb, key, qp);
334 }
335
bnxt_re_fill_ppp(struct bnxt_re_push_buffer * pbuf,struct bnxt_re_qp * qp,uint8_t len,uint32_t idx)336 void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf,
337 struct bnxt_re_qp *qp, uint8_t len, uint32_t idx)
338 {
339 struct bnxt_re_db_ppp_hdr phdr = {};
340 __u64 *dst, *src;
341 __u8 plen;
342 int indx;
343
344 src = (__u64 *)&phdr;
345 plen = len + sizeof(phdr) + bnxt_re_get_sqe_hdr_sz();
346
347 bnxt_re_init_db_hdr(&phdr.db_hdr,
348 (*qp->jsqq->hwque->dbtail |
349 ((qp->jsqq->hwque->flags &
350 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
351 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
352 qp->qpid,
353 BNXT_RE_QUE_TYPE_SQ);
354
355 phdr.rsv_psz_pidx = ((pbuf->st_idx & BNXT_RE_DB_INDX_MASK) |
356 (((plen % 8 ? (plen / 8) + 1 :
357 plen / 8) & BNXT_RE_PUSH_SIZE_MASK) <<
358 BNXT_RE_PUSH_SIZE_SHIFT));
359
360 bnxt_re_wm_barrier();
361 for (indx = 0; indx < 2; indx++) {
362 dst = (__u64 *)(pbuf->pbuf + indx);
363 iowrite64(dst, *src);
364 src++;
365 }
366 bnxt_re_copy_data_to_pb(pbuf, 1, idx);
367 mmio_flush_writes();
368 }
369
bnxt_re_fill_push_wcb(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf,uint32_t idx)370 void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp,
371 struct bnxt_re_push_buffer *pbuf, uint32_t idx)
372 {
373 bnxt_re_ring_pstart_db(qp, pbuf);
374 mmio_wc_start();
375 bnxt_re_copy_data_to_pb(pbuf, 0, idx);
376 /* Flush WQE write before push end db. */
377 mmio_flush_writes();
378 bnxt_re_ring_pend_db(qp, pbuf);
379 }
380
bnxt_re_init_pbuf_list(struct bnxt_re_context * ucntx)381 int bnxt_re_init_pbuf_list(struct bnxt_re_context *ucntx)
382 {
383 struct bnxt_re_push_buffer *pbuf;
384 int indx, wqesz;
385 int size, offt;
386 __u64 wcpage;
387 __u64 dbpage;
388 void *base;
389
390 size = (sizeof(*ucntx->pbrec) +
391 16 * (sizeof(*ucntx->pbrec->pbuf) +
392 sizeof(struct bnxt_re_push_wqe)));
393 ucntx->pbrec = calloc(1, size);
394 if (!ucntx->pbrec)
395 goto out;
396
397 offt = sizeof(*ucntx->pbrec);
398 base = ucntx->pbrec;
399 ucntx->pbrec->pbuf = (base + offt);
400 ucntx->pbrec->pbmap = ~0x00;
401 ucntx->pbrec->pbmap &= ~0x7fff; /* 15 bits */
402 ucntx->pbrec->udpi = &ucntx->udpi;
403
404 wqesz = sizeof(struct bnxt_re_push_wqe);
405 wcpage = (__u64)ucntx->udpi.wcdbpg;
406 dbpage = (__u64)ucntx->udpi.dbpage;
407 offt = sizeof(*ucntx->pbrec->pbuf) * 16;
408 base = (char *)ucntx->pbrec->pbuf + offt;
409 for (indx = 0; indx < 16; indx++) {
410 pbuf = &ucntx->pbrec->pbuf[indx];
411 pbuf->wqe = base + indx * wqesz;
412 pbuf->pbuf = (__u64 *)(wcpage + indx * wqesz);
413 pbuf->ucdb = (__u64 *)(dbpage + (indx + 1) * sizeof(__u64));
414 pbuf->wcdpi = ucntx->udpi.wcdpi;
415 }
416
417 return 0;
418 out:
419 return -ENOMEM;
420 }
421
bnxt_re_get_pbuf(uint8_t * push_st_en,uint8_t ppp_idx,struct bnxt_re_context * cntx)422 struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en,
423 uint8_t ppp_idx,
424 struct bnxt_re_context *cntx)
425 {
426 struct bnxt_re_push_buffer *pbuf = NULL;
427 uint8_t buf_state = 0;
428 __u32 old;
429 int bit;
430
431 if (_is_chip_thor2(cntx->cctx)) {
432 buf_state = !!(*push_st_en & BNXT_RE_PPP_STATE_MASK);
433 pbuf = &cntx->pbrec->pbuf[(ppp_idx * 2) + buf_state];
434 /* Flip */
435 *push_st_en ^= 1UL << BNXT_RE_PPP_ST_SHIFT;
436 } else {
437 old = cntx->pbrec->pbmap;
438 while ((bit = __builtin_ffs(~cntx->pbrec->pbmap)) != 0) {
439 if (__sync_bool_compare_and_swap
440 (&cntx->pbrec->pbmap,
441 old,
442 (old | 0x01 << (bit - 1))))
443 break;
444 old = cntx->pbrec->pbmap;
445 }
446
447 if (bit) {
448 pbuf = &cntx->pbrec->pbuf[bit];
449 pbuf->nbit = bit;
450 }
451 }
452
453 return pbuf;
454 }
455
bnxt_re_put_pbuf(struct bnxt_re_context * cntx,struct bnxt_re_push_buffer * pbuf)456 void bnxt_re_put_pbuf(struct bnxt_re_context *cntx,
457 struct bnxt_re_push_buffer *pbuf)
458 {
459 struct bnxt_re_push_rec *pbrec;
460 __u32 old;
461 int bit;
462
463 if (_is_chip_thor2(cntx->cctx))
464 return;
465
466 pbrec = cntx->pbrec;
467
468 if (pbuf->nbit) {
469 bit = pbuf->nbit;
470 pbuf->nbit = 0;
471 old = pbrec->pbmap;
472 while (!__sync_bool_compare_and_swap(&pbrec->pbmap, old,
473 (old & (~(0x01 <<
474 (bit - 1))))))
475 old = pbrec->pbmap;
476 }
477 }
478
bnxt_re_destroy_pbuf_list(struct bnxt_re_context * cntx)479 void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx)
480 {
481 free(cntx->pbrec);
482 }
483
bnxt_re_replay_db(struct bnxt_re_context * cntx,struct xorshift32_state * state,struct bnxt_re_dpi * dpi,uint64_t * shadow_key,uint8_t * dbr_lock)484 void bnxt_re_replay_db(struct bnxt_re_context *cntx,
485 struct xorshift32_state *state, struct bnxt_re_dpi *dpi,
486 uint64_t *shadow_key, uint8_t *dbr_lock)
487 {
488 if (bnxt_re_do_pacing(cntx, state))
489 return;
490 cntx->replay_cnt++;
491 if (cntx->replay_cnt % BNXT_RE_DB_REPLAY_YIELD_CNT == 0)
492 pthread_yield();
493 if (__sync_bool_compare_and_swap(dbr_lock, 0, 1)) {
494 bnxt_re_wm_barrier();
495 if (*shadow_key == BNXT_RE_DB_KEY_INVALID) {
496 *dbr_lock = 0;
497 return;
498 }
499 iowrite64(dpi->dbpage, *shadow_key);
500 bnxt_re_wm_barrier();
501 *dbr_lock = 0;
502 }
503 }
504
bnxt_re_db_recovery(struct bnxt_re_context * cntx)505 void bnxt_re_db_recovery(struct bnxt_re_context *cntx)
506 {
507 struct bnxt_re_list_node *cur, *tmp;
508 struct bnxt_re_qp *qp;
509 struct bnxt_re_cq *cq;
510 struct bnxt_re_srq *srq;
511
512 pthread_spin_lock(&cntx->qp_dbr_res.lock);
513 list_for_each_node_safe(cur, tmp, &cntx->qp_dbr_res.head) {
514 qp = list_node(cur, struct bnxt_re_qp, dbnode);
515 bnxt_re_replay_db(cntx, &qp->rand, qp->udpi,
516 &qp->sq_shadow_db_key, &qp->sq_dbr_lock);
517 bnxt_re_replay_db(cntx, &qp->rand, qp->udpi,
518 &qp->rq_shadow_db_key, &qp->rq_dbr_lock);
519 }
520 pthread_spin_unlock(&cntx->qp_dbr_res.lock);
521 pthread_spin_lock(&cntx->cq_dbr_res.lock);
522 list_for_each_node_safe(cur, tmp, &cntx->cq_dbr_res.head) {
523 cq = list_node(cur, struct bnxt_re_cq, dbnode);
524 bnxt_re_replay_db(cntx, &cq->rand, cq->udpi,
525 &cq->shadow_db_key, &cq->dbr_lock);
526 }
527 pthread_spin_unlock(&cntx->cq_dbr_res.lock);
528 pthread_spin_lock(&cntx->srq_dbr_res.lock);
529 list_for_each_node_safe(cur, tmp, &cntx->srq_dbr_res.head) {
530 srq = list_node(cur, struct bnxt_re_srq, dbnode);
531 bnxt_re_replay_db(cntx, &srq->rand, srq->udpi,
532 &srq->shadow_db_key, &srq->dbr_lock);
533 }
534 pthread_spin_unlock(&cntx->srq_dbr_res.lock);
535 }
536
bnxt_re_dbr_thread(void * arg)537 void *bnxt_re_dbr_thread(void *arg)
538 {
539 uint32_t *epoch, *epoch_ack, usr_epoch;
540 struct bnxt_re_context *cntx = arg;
541 struct ibv_cq *ev_cq;
542 void *ev_ctx;
543 int ret;
544
545 while (1) {
546 ret = ibv_get_cq_event(cntx->dbr_ev_chan, &ev_cq, &ev_ctx);
547 if (ret) {
548 fprintf(stderr, "Failed to get cq_event\n");
549 pthread_exit(NULL);
550 }
551 epoch = cntx->db_recovery_page;
552 epoch_ack = epoch + 1;
553 if (!epoch || !epoch_ack) {
554 fprintf(stderr, "DB reovery page is NULL\n");
555 pthread_exit(NULL);
556 }
557 if (*epoch == *epoch_ack) {
558 ibv_ack_cq_events(ev_cq, 1);
559 continue;
560 }
561 usr_epoch = *epoch;
562 bnxt_re_db_recovery(cntx);
563 *epoch_ack = usr_epoch;
564 ibv_ack_cq_events(ev_cq, 1);
565 }
566 }
567