1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD: stable/12/sys/dev/ena/ena_netmap.c 372883 2023-01-31 13:19:16Z mw $");
32
33 #ifdef DEV_NETMAP
34
35 #include "ena.h"
36 #include "ena_netmap.h"
37
38 #define ENA_NETMAP_MORE_FRAMES 1
39 #define ENA_NETMAP_NO_MORE_FRAMES 0
40 #define ENA_MAX_FRAMES 16384
41
42 struct ena_netmap_ctx {
43 struct netmap_kring *kring;
44 struct ena_adapter *adapter;
45 struct netmap_adapter *na;
46 struct netmap_slot *slots;
47 struct ena_ring *ring;
48 struct ena_com_io_cq *io_cq;
49 struct ena_com_io_sq *io_sq;
50 u_int nm_i;
51 uint16_t nt;
52 uint16_t lim;
53 };
54
55 /* Netmap callbacks */
56 static int ena_netmap_reg(struct netmap_adapter *, int);
57 static int ena_netmap_txsync(struct netmap_kring *, int);
58 static int ena_netmap_rxsync(struct netmap_kring *, int);
59
60 /* Helper functions */
61 static int ena_netmap_tx_frames(struct ena_netmap_ctx *);
62 static int ena_netmap_tx_frame(struct ena_netmap_ctx *);
63 static inline uint16_t ena_netmap_count_slots(struct ena_netmap_ctx *);
64 static inline uint16_t ena_netmap_packet_len(struct netmap_slot *, u_int,
65 uint16_t);
66 static int ena_netmap_copy_data(struct netmap_adapter *, struct netmap_slot *,
67 u_int, uint16_t, uint16_t, void *);
68 static int ena_netmap_map_single_slot(struct netmap_adapter *,
69 struct netmap_slot *, bus_dma_tag_t, bus_dmamap_t, void **, uint64_t *);
70 static int ena_netmap_tx_map_slots(struct ena_netmap_ctx *,
71 struct ena_tx_buffer *, void **, uint16_t *, uint16_t *);
72 static void ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *,
73 struct ena_tx_buffer *);
74 static void ena_netmap_tx_cleanup(struct ena_netmap_ctx *);
75 static uint16_t ena_netmap_tx_clean_one(struct ena_netmap_ctx *, uint16_t);
76 static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
77 static int ena_netmap_rx_frames(struct ena_netmap_ctx *);
78 static int ena_netmap_rx_frame(struct ena_netmap_ctx *);
79 static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t, int *);
80 static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *);
81 static void ena_netmap_fill_ctx(struct netmap_kring *, struct ena_netmap_ctx *,
82 uint16_t);
83
84 int
ena_netmap_attach(struct ena_adapter * adapter)85 ena_netmap_attach(struct ena_adapter *adapter)
86 {
87 struct netmap_adapter na;
88
89 ena_log_nm(adapter->pdev, INFO, "netmap attach\n");
90
91 bzero(&na, sizeof(na));
92 na.na_flags = NAF_MOREFRAG;
93 na.ifp = adapter->ifp;
94 na.num_tx_desc = adapter->requested_tx_ring_size;
95 na.num_rx_desc = adapter->requested_rx_ring_size;
96 na.num_tx_rings = adapter->num_io_queues;
97 na.num_rx_rings = adapter->num_io_queues;
98 na.rx_buf_maxsize = adapter->buf_ring_size;
99 na.nm_txsync = ena_netmap_txsync;
100 na.nm_rxsync = ena_netmap_rxsync;
101 na.nm_register = ena_netmap_reg;
102
103 return (netmap_attach(&na));
104 }
105
106 int
ena_netmap_alloc_rx_slot(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)107 ena_netmap_alloc_rx_slot(struct ena_adapter *adapter, struct ena_ring *rx_ring,
108 struct ena_rx_buffer *rx_info)
109 {
110 struct netmap_adapter *na = NA(adapter->ifp);
111 struct netmap_kring *kring;
112 struct netmap_ring *ring;
113 struct netmap_slot *slot;
114 void *addr;
115 uint64_t paddr;
116 int nm_i, qid, head, lim, rc;
117
118 /* if previously allocated frag is not used */
119 if (unlikely(rx_info->netmap_buf_idx != 0))
120 return (0);
121
122 qid = rx_ring->qid;
123 kring = na->rx_rings[qid];
124 nm_i = kring->nr_hwcur;
125 head = kring->rhead;
126
127 ena_log_nm(adapter->pdev, DBG,
128 "nr_hwcur: %d, nr_hwtail: %d, rhead: %d, rcur: %d, rtail: %d\n",
129 kring->nr_hwcur, kring->nr_hwtail, kring->rhead, kring->rcur,
130 kring->rtail);
131
132 if ((nm_i == head) && rx_ring->initialized) {
133 ena_log_nm(adapter->pdev, ERR,
134 "No free slots in netmap ring\n");
135 return (ENOMEM);
136 }
137
138 ring = kring->ring;
139 if (ring == NULL) {
140 ena_log_nm(adapter->pdev, ERR, "Rx ring %d is NULL\n", qid);
141 return (EFAULT);
142 }
143 slot = &ring->slot[nm_i];
144
145 addr = PNMB(na, slot, &paddr);
146 if (addr == NETMAP_BUF_BASE(na)) {
147 ena_log_nm(adapter->pdev, ERR, "Bad buff in slot\n");
148 return (EFAULT);
149 }
150
151 rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
152 if (rc != 0) {
153 ena_log_nm(adapter->pdev, WARN, "DMA mapping error\n");
154 return (rc);
155 }
156 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
157
158 rx_info->ena_buf.paddr = paddr;
159 rx_info->ena_buf.len = ring->nr_buf_size;
160 rx_info->mbuf = NULL;
161 rx_info->netmap_buf_idx = slot->buf_idx;
162
163 slot->buf_idx = 0;
164
165 lim = kring->nkr_num_slots - 1;
166 kring->nr_hwcur = nm_next(nm_i, lim);
167
168 return (0);
169 }
170
171 void
ena_netmap_free_rx_slot(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)172 ena_netmap_free_rx_slot(struct ena_adapter *adapter, struct ena_ring *rx_ring,
173 struct ena_rx_buffer *rx_info)
174 {
175 struct netmap_adapter *na;
176 struct netmap_kring *kring;
177 struct netmap_slot *slot;
178 int nm_i, qid, lim;
179
180 na = NA(adapter->ifp);
181 if (na == NULL) {
182 ena_log_nm(adapter->pdev, ERR, "netmap adapter is NULL\n");
183 return;
184 }
185
186 if (na->rx_rings == NULL) {
187 ena_log_nm(adapter->pdev, ERR, "netmap rings are NULL\n");
188 return;
189 }
190
191 qid = rx_ring->qid;
192 kring = na->rx_rings[qid];
193 if (kring == NULL) {
194 ena_log_nm(adapter->pdev, ERR,
195 "netmap kernel ring %d is NULL\n", qid);
196 return;
197 }
198
199 lim = kring->nkr_num_slots - 1;
200 nm_i = nm_prev(kring->nr_hwcur, lim);
201
202 if (kring->nr_mode != NKR_NETMAP_ON)
203 return;
204
205 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
206 BUS_DMASYNC_POSTREAD);
207 netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
208
209 KASSERT(kring->ring != NULL, ("Netmap Rx ring is NULL\n"));
210
211 slot = &kring->ring->slot[nm_i];
212
213 ENA_WARN(slot->buf_idx != 0, adapter->ena_dev, "Overwrite slot buf\n");
214 slot->buf_idx = rx_info->netmap_buf_idx;
215 slot->flags = NS_BUF_CHANGED;
216
217 rx_info->netmap_buf_idx = 0;
218 kring->nr_hwcur = nm_i;
219 }
220
221 static bool
ena_ring_in_netmap(struct ena_adapter * adapter,int qid,enum txrx x)222 ena_ring_in_netmap(struct ena_adapter *adapter, int qid, enum txrx x)
223 {
224 struct netmap_adapter *na;
225 struct netmap_kring *kring;
226
227 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
228 na = NA(adapter->ifp);
229 kring = (x == NR_RX) ? na->rx_rings[qid] : na->tx_rings[qid];
230 if (kring->nr_mode == NKR_NETMAP_ON)
231 return true;
232 }
233 return false;
234 }
235
236 bool
ena_tx_ring_in_netmap(struct ena_adapter * adapter,int qid)237 ena_tx_ring_in_netmap(struct ena_adapter *adapter, int qid)
238 {
239 return ena_ring_in_netmap(adapter, qid, NR_TX);
240 }
241
242 bool
ena_rx_ring_in_netmap(struct ena_adapter * adapter,int qid)243 ena_rx_ring_in_netmap(struct ena_adapter *adapter, int qid)
244 {
245 return ena_ring_in_netmap(adapter, qid, NR_RX);
246 }
247
248 static void
ena_netmap_reset_ring(struct ena_adapter * adapter,int qid,enum txrx x)249 ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
250 {
251 if (!ena_ring_in_netmap(adapter, qid, x))
252 return;
253
254 netmap_reset(NA(adapter->ifp), x, qid, 0);
255 ena_log_nm(adapter->pdev, INFO, "%s ring %d is in netmap mode\n",
256 (x == NR_TX) ? "Tx" : "Rx", qid);
257 }
258
259 void
ena_netmap_reset_rx_ring(struct ena_adapter * adapter,int qid)260 ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
261 {
262 ena_netmap_reset_ring(adapter, qid, NR_RX);
263 }
264
265 void
ena_netmap_reset_tx_ring(struct ena_adapter * adapter,int qid)266 ena_netmap_reset_tx_ring(struct ena_adapter *adapter, int qid)
267 {
268 ena_netmap_reset_ring(adapter, qid, NR_TX);
269 }
270
271 static int
ena_netmap_reg(struct netmap_adapter * na,int onoff)272 ena_netmap_reg(struct netmap_adapter *na, int onoff)
273 {
274 struct ifnet *ifp = na->ifp;
275 struct ena_adapter *adapter = ifp->if_softc;
276 device_t pdev = adapter->pdev;
277 struct netmap_kring *kring;
278 enum txrx t;
279 int rc, i;
280
281 ENA_LOCK_LOCK();
282 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
283 ena_down(adapter);
284
285 if (onoff) {
286 ena_log_nm(pdev, INFO, "netmap on\n");
287 for_rx_tx(t) {
288 for (i = 0; i <= nma_get_nrings(na, t); i++) {
289 kring = NMR(na, t)[i];
290 if (nm_kring_pending_on(kring)) {
291 kring->nr_mode = NKR_NETMAP_ON;
292 }
293 }
294 }
295 nm_set_native_flags(na);
296 } else {
297 ena_log_nm(pdev, INFO, "netmap off\n");
298 nm_clear_native_flags(na);
299 for_rx_tx(t) {
300 for (i = 0; i <= nma_get_nrings(na, t); i++) {
301 kring = NMR(na, t)[i];
302 if (nm_kring_pending_off(kring)) {
303 kring->nr_mode = NKR_NETMAP_OFF;
304 }
305 }
306 }
307 }
308
309 rc = ena_up(adapter);
310 if (rc != 0) {
311 ena_log_nm(pdev, WARN, "ena_up failed with rc=%d\n", rc);
312 adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
313 nm_clear_native_flags(na);
314 ena_destroy_device(adapter, false);
315 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
316 rc = ena_restore_device(adapter);
317 }
318 ENA_LOCK_UNLOCK();
319
320 return (rc);
321 }
322
323 static int
ena_netmap_txsync(struct netmap_kring * kring,int flags)324 ena_netmap_txsync(struct netmap_kring *kring, int flags)
325 {
326 struct ena_netmap_ctx ctx;
327 int rc = 0;
328
329 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_TXQ_IDX(kring->ring_id));
330 ctx.ring = &ctx.adapter->tx_ring[kring->ring_id];
331
332 ENA_RING_MTX_LOCK(ctx.ring);
333 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter)))
334 goto txsync_end;
335
336 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
337 goto txsync_end;
338
339 rc = ena_netmap_tx_frames(&ctx);
340 ena_netmap_tx_cleanup(&ctx);
341
342 txsync_end:
343 ENA_RING_MTX_UNLOCK(ctx.ring);
344 return (rc);
345 }
346
347 static int
ena_netmap_tx_frames(struct ena_netmap_ctx * ctx)348 ena_netmap_tx_frames(struct ena_netmap_ctx *ctx)
349 {
350 struct ena_ring *tx_ring = ctx->ring;
351 int rc = 0;
352
353 ctx->nm_i = ctx->kring->nr_hwcur;
354 ctx->nt = ctx->ring->next_to_use;
355
356 __builtin_prefetch(&ctx->slots[ctx->nm_i]);
357
358 while (ctx->nm_i != ctx->kring->rhead) {
359 if ((rc = ena_netmap_tx_frame(ctx)) != 0) {
360 /*
361 * When there is no empty space in Tx ring, error is
362 * still being returned. It should not be passed to the
363 * netmap, as application knows current ring state from
364 * netmap ring pointers. Returning error there could
365 * cause application to exit, but the Tx ring is
366 * commonly being full.
367 */
368 if (rc == ENA_COM_NO_MEM)
369 rc = 0;
370 break;
371 }
372 tx_ring->acum_pkts++;
373 }
374
375 /* If any packet was sent... */
376 if (likely(ctx->nm_i != ctx->kring->nr_hwcur)) {
377 /* ...send the doorbell to the device. */
378 ena_ring_tx_doorbell(tx_ring);
379
380 ctx->ring->next_to_use = ctx->nt;
381 ctx->kring->nr_hwcur = ctx->nm_i;
382 }
383
384 return (rc);
385 }
386
387 static int
ena_netmap_tx_frame(struct ena_netmap_ctx * ctx)388 ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
389 {
390 struct ena_com_tx_ctx ena_tx_ctx;
391 struct ena_adapter *adapter;
392 struct ena_ring *tx_ring;
393 struct ena_tx_buffer *tx_info;
394 uint16_t req_id;
395 uint16_t header_len;
396 uint16_t packet_len;
397 int nb_hw_desc;
398 int rc;
399 void *push_hdr;
400
401 adapter = ctx->adapter;
402 if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
403 ena_log_nm(adapter->pdev, WARN, "Too many slots per packet\n");
404 return (EINVAL);
405 }
406
407 tx_ring = ctx->ring;
408
409 req_id = tx_ring->free_tx_ids[ctx->nt];
410 tx_info = &tx_ring->tx_buffer_info[req_id];
411 tx_info->num_of_bufs = 0;
412 tx_info->nm_info.sockets_used = 0;
413
414 rc = ena_netmap_tx_map_slots(ctx, tx_info, &push_hdr, &header_len,
415 &packet_len);
416 if (unlikely(rc != 0)) {
417 ena_log_nm(adapter->pdev, ERR, "Failed to map Tx slot\n");
418 return (rc);
419 }
420
421 bzero(&ena_tx_ctx, sizeof(struct ena_com_tx_ctx));
422 ena_tx_ctx.ena_bufs = tx_info->bufs;
423 ena_tx_ctx.push_header = push_hdr;
424 ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
425 ena_tx_ctx.req_id = req_id;
426 ena_tx_ctx.header_len = header_len;
427 ena_tx_ctx.meta_valid = adapter->disable_meta_caching;
428
429 /* There are no any offloads, as the netmap doesn't support them */
430
431 if (tx_ring->acum_pkts == ENA_DB_THRESHOLD ||
432 ena_com_is_doorbell_needed(ctx->io_sq, &ena_tx_ctx))
433 ena_ring_tx_doorbell(tx_ring);
434
435 rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
436 if (unlikely(rc != 0)) {
437 if (likely(rc == ENA_COM_NO_MEM)) {
438 ena_log_nm(adapter->pdev, DBG,
439 "Tx ring[%d] is out of space\n", tx_ring->que->id);
440 } else {
441 ena_log_nm(adapter->pdev, ERR,
442 "Failed to prepare Tx bufs\n");
443 ena_trigger_reset(adapter,
444 ENA_REGS_RESET_DRIVER_INVALID_STATE);
445 }
446 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
447
448 ena_netmap_unmap_last_socket_chain(ctx, tx_info);
449 return (rc);
450 }
451
452 counter_enter();
453 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
454 counter_u64_add_protected(tx_ring->tx_stats.bytes, packet_len);
455 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
456 counter_u64_add_protected(adapter->hw_stats.tx_bytes, packet_len);
457 counter_exit();
458
459 tx_info->tx_descs = nb_hw_desc;
460
461 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
462
463 for (unsigned int i = 0; i < tx_info->num_of_bufs; i++)
464 bus_dmamap_sync(adapter->tx_buf_tag,
465 tx_info->nm_info.map_seg[i], BUS_DMASYNC_PREWRITE);
466
467 return (0);
468 }
469
470 static inline uint16_t
ena_netmap_count_slots(struct ena_netmap_ctx * ctx)471 ena_netmap_count_slots(struct ena_netmap_ctx *ctx)
472 {
473 uint16_t slots = 1;
474 uint16_t nm = ctx->nm_i;
475
476 while ((ctx->slots[nm].flags & NS_MOREFRAG) != 0) {
477 slots++;
478 nm = nm_next(nm, ctx->lim);
479 }
480
481 return slots;
482 }
483
484 static inline uint16_t
ena_netmap_packet_len(struct netmap_slot * slots,u_int slot_index,uint16_t limit)485 ena_netmap_packet_len(struct netmap_slot *slots, u_int slot_index,
486 uint16_t limit)
487 {
488 struct netmap_slot *nm_slot;
489 uint16_t packet_size = 0;
490
491 do {
492 nm_slot = &slots[slot_index];
493 packet_size += nm_slot->len;
494 slot_index = nm_next(slot_index, limit);
495 } while ((nm_slot->flags & NS_MOREFRAG) != 0);
496
497 return packet_size;
498 }
499
500 static int
ena_netmap_copy_data(struct netmap_adapter * na,struct netmap_slot * slots,u_int slot_index,uint16_t limit,uint16_t bytes_to_copy,void * destination)501 ena_netmap_copy_data(struct netmap_adapter *na, struct netmap_slot *slots,
502 u_int slot_index, uint16_t limit, uint16_t bytes_to_copy, void *destination)
503 {
504 struct netmap_slot *nm_slot;
505 void *slot_vaddr;
506 uint16_t data_amount;
507
508 do {
509 nm_slot = &slots[slot_index];
510 slot_vaddr = NMB(na, nm_slot);
511 if (unlikely(slot_vaddr == NULL))
512 return (EINVAL);
513
514 data_amount = min_t(uint16_t, bytes_to_copy, nm_slot->len);
515 memcpy(destination, slot_vaddr, data_amount);
516 bytes_to_copy -= data_amount;
517
518 slot_index = nm_next(slot_index, limit);
519 } while ((nm_slot->flags & NS_MOREFRAG) != 0 && bytes_to_copy > 0);
520
521 return (0);
522 }
523
524 static int
ena_netmap_map_single_slot(struct netmap_adapter * na,struct netmap_slot * slot,bus_dma_tag_t dmatag,bus_dmamap_t dmamap,void ** vaddr,uint64_t * paddr)525 ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
526 bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
527 {
528 device_t pdev;
529 int rc;
530
531 pdev = ((struct ena_adapter *)na->ifp->if_softc)->pdev;
532
533 *vaddr = PNMB(na, slot, paddr);
534 if (unlikely(vaddr == NULL)) {
535 ena_log_nm(pdev, ERR, "Slot address is NULL\n");
536 return (EINVAL);
537 }
538
539 rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
540 if (unlikely(rc != 0)) {
541 ena_log_nm(pdev, ERR, "Failed to map slot %d for DMA\n",
542 slot->buf_idx);
543 return (EINVAL);
544 }
545
546 return (0);
547 }
548
549 static int
ena_netmap_tx_map_slots(struct ena_netmap_ctx * ctx,struct ena_tx_buffer * tx_info,void ** push_hdr,uint16_t * header_len,uint16_t * packet_len)550 ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx,
551 struct ena_tx_buffer *tx_info, void **push_hdr, uint16_t *header_len,
552 uint16_t *packet_len)
553 {
554 struct netmap_slot *slot;
555 struct ena_com_buf *ena_buf;
556 struct ena_adapter *adapter;
557 struct ena_ring *tx_ring;
558 struct ena_netmap_tx_info *nm_info;
559 bus_dmamap_t *nm_maps;
560 void *vaddr;
561 uint64_t paddr;
562 uint32_t *nm_buf_idx;
563 uint32_t slot_head_len;
564 uint32_t frag_len;
565 uint32_t remaining_len;
566 uint16_t push_len;
567 uint16_t delta;
568 int rc;
569
570 adapter = ctx->adapter;
571 tx_ring = ctx->ring;
572 ena_buf = tx_info->bufs;
573 nm_info = &tx_info->nm_info;
574 nm_maps = nm_info->map_seg;
575 nm_buf_idx = nm_info->socket_buf_idx;
576 slot = &ctx->slots[ctx->nm_i];
577
578 slot_head_len = slot->len;
579 *packet_len = ena_netmap_packet_len(ctx->slots, ctx->nm_i, ctx->lim);
580 remaining_len = *packet_len;
581 delta = 0;
582
583 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
584 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
585 /*
586 * When the device is in LLQ mode, the driver will copy
587 * the header into the device memory space.
588 * The ena_com layer assumes that the header is in a linear
589 * memory space.
590 * This assumption might be wrong since part of the header
591 * can be in the fragmented buffers.
592 * First, check if header fits in the first slot. If not, copy
593 * it to separate buffer that will be holding linearized data.
594 */
595 push_len = min_t(uint32_t, *packet_len,
596 tx_ring->tx_max_header_size);
597 *header_len = push_len;
598 /* If header is in linear space, just point to socket's data. */
599 if (likely(push_len <= slot_head_len)) {
600 *push_hdr = NMB(ctx->na, slot);
601 if (unlikely(push_hdr == NULL)) {
602 ena_log_nm(adapter->pdev, ERR,
603 "Slot vaddress is NULL\n");
604 return (EINVAL);
605 }
606 /*
607 * Otherwise, copy whole portion of header from multiple
608 * slots to intermediate buffer.
609 */
610 } else {
611 rc = ena_netmap_copy_data(ctx->na, ctx->slots,
612 ctx->nm_i, ctx->lim, push_len,
613 tx_ring->push_buf_intermediate_buf);
614 if (unlikely(rc)) {
615 ena_log_nm(adapter->pdev, ERR,
616 "Failed to copy data from slots to push_buf\n");
617 return (EINVAL);
618 }
619
620 *push_hdr = tx_ring->push_buf_intermediate_buf;
621 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
622
623 delta = push_len - slot_head_len;
624 }
625
626 ena_log_nm(adapter->pdev, DBG,
627 "slot: %d header_buf->vaddr: %p push_len: %d\n",
628 slot->buf_idx, *push_hdr, push_len);
629
630 /*
631 * If header was in linear memory space, map for the dma rest of
632 * the data in the first mbuf of the mbuf chain.
633 */
634 if (slot_head_len > push_len) {
635 rc = ena_netmap_map_single_slot(ctx->na, slot,
636 adapter->tx_buf_tag, *nm_maps, &vaddr, &paddr);
637 if (unlikely(rc != 0)) {
638 ena_log_nm(adapter->pdev, ERR,
639 "DMA mapping error\n");
640 return (rc);
641 }
642 nm_maps++;
643
644 ena_buf->paddr = paddr + push_len;
645 ena_buf->len = slot->len - push_len;
646 ena_buf++;
647
648 tx_info->num_of_bufs++;
649 }
650
651 remaining_len -= slot->len;
652
653 /* Save buf idx before advancing */
654 *nm_buf_idx = slot->buf_idx;
655 nm_buf_idx++;
656 slot->buf_idx = 0;
657
658 /* Advance to the next socket */
659 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
660 slot = &ctx->slots[ctx->nm_i];
661 nm_info->sockets_used++;
662
663 /*
664 * If header is in non linear space (delta > 0), then skip mbufs
665 * containing header and map the last one containing both header
666 * and the packet data.
667 * The first segment is already counted in.
668 */
669 while (delta > 0) {
670 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
671 frag_len = slot->len;
672
673 /*
674 * If whole segment contains header just move to the
675 * next one and reduce delta.
676 */
677 if (unlikely(delta >= frag_len)) {
678 delta -= frag_len;
679 } else {
680 /*
681 * Map the data and then assign it with the
682 * offsets
683 */
684 rc = ena_netmap_map_single_slot(ctx->na, slot,
685 adapter->tx_buf_tag, *nm_maps, &vaddr,
686 &paddr);
687 if (unlikely(rc != 0)) {
688 ena_log_nm(adapter->pdev, ERR,
689 "DMA mapping error\n");
690 goto error_map;
691 }
692 nm_maps++;
693
694 ena_buf->paddr = paddr + delta;
695 ena_buf->len = slot->len - delta;
696 ena_buf++;
697
698 tx_info->num_of_bufs++;
699 delta = 0;
700 }
701
702 remaining_len -= slot->len;
703
704 /* Save buf idx before advancing */
705 *nm_buf_idx = slot->buf_idx;
706 nm_buf_idx++;
707 slot->buf_idx = 0;
708
709 /* Advance to the next socket */
710 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
711 slot = &ctx->slots[ctx->nm_i];
712 nm_info->sockets_used++;
713 }
714 } else {
715 *push_hdr = NULL;
716 /*
717 * header_len is just a hint for the device. Because netmap is
718 * not giving us any information about packet header length and
719 * it is not guaranteed that all packet headers will be in the
720 * 1st slot, setting header_len to 0 is making the device ignore
721 * this value and resolve header on it's own.
722 */
723 *header_len = 0;
724 }
725
726 /* Map all remaining data (regular routine for non-LLQ mode) */
727 while (remaining_len > 0) {
728 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
729
730 rc = ena_netmap_map_single_slot(ctx->na, slot,
731 adapter->tx_buf_tag, *nm_maps, &vaddr, &paddr);
732 if (unlikely(rc != 0)) {
733 ena_log_nm(adapter->pdev, ERR, "DMA mapping error\n");
734 goto error_map;
735 }
736 nm_maps++;
737
738 ena_buf->paddr = paddr;
739 ena_buf->len = slot->len;
740 ena_buf++;
741
742 tx_info->num_of_bufs++;
743
744 remaining_len -= slot->len;
745
746 /* Save buf idx before advancing */
747 *nm_buf_idx = slot->buf_idx;
748 nm_buf_idx++;
749 slot->buf_idx = 0;
750
751 /* Advance to the next socket */
752 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
753 slot = &ctx->slots[ctx->nm_i];
754 nm_info->sockets_used++;
755 }
756
757 return (0);
758
759 error_map:
760 ena_netmap_unmap_last_socket_chain(ctx, tx_info);
761
762 return (rc);
763 }
764
765 static void
ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx * ctx,struct ena_tx_buffer * tx_info)766 ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *ctx,
767 struct ena_tx_buffer *tx_info)
768 {
769 struct ena_netmap_tx_info *nm_info;
770 int n;
771
772 nm_info = &tx_info->nm_info;
773
774 /**
775 * As the used sockets must not be equal to the buffers used in the LLQ
776 * mode, they must be treated separately.
777 * First, unmap the DMA maps.
778 */
779 n = tx_info->num_of_bufs;
780 while (n--) {
781 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
782 nm_info->map_seg[n]);
783 }
784 tx_info->num_of_bufs = 0;
785
786 /* Next, retain the sockets back to the userspace */
787 n = nm_info->sockets_used;
788 while (n--) {
789 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
790 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
791 nm_info->socket_buf_idx[n] = 0;
792 ctx->nm_i = nm_prev(ctx->nm_i, ctx->lim);
793 }
794 nm_info->sockets_used = 0;
795 }
796
797 static void
ena_netmap_tx_cleanup(struct ena_netmap_ctx * ctx)798 ena_netmap_tx_cleanup(struct ena_netmap_ctx *ctx)
799 {
800 uint16_t req_id;
801 uint16_t total_tx_descs = 0;
802
803 ctx->nm_i = ctx->kring->nr_hwtail;
804 ctx->nt = ctx->ring->next_to_clean;
805
806 /* Reclaim buffers for completed transmissions */
807 while (ena_com_tx_comp_req_id_get(ctx->io_cq, &req_id) >= 0) {
808 if (validate_tx_req_id(ctx->ring, req_id) != 0)
809 break;
810 total_tx_descs += ena_netmap_tx_clean_one(ctx, req_id);
811 }
812
813 ctx->kring->nr_hwtail = ctx->nm_i;
814
815 if (total_tx_descs > 0) {
816 /* acknowledge completion of sent packets */
817 ctx->ring->next_to_clean = ctx->nt;
818 ena_com_comp_ack(ctx->ring->ena_com_io_sq, total_tx_descs);
819 ena_com_update_dev_comp_head(ctx->ring->ena_com_io_cq);
820 }
821 }
822
823 static uint16_t
ena_netmap_tx_clean_one(struct ena_netmap_ctx * ctx,uint16_t req_id)824 ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id)
825 {
826 struct ena_tx_buffer *tx_info;
827 struct ena_netmap_tx_info *nm_info;
828 int n;
829
830 tx_info = &ctx->ring->tx_buffer_info[req_id];
831 nm_info = &tx_info->nm_info;
832
833 /**
834 * As the used sockets must not be equal to the buffers used in the LLQ
835 * mode, they must be treated separately.
836 * First, unmap the DMA maps.
837 */
838 n = tx_info->num_of_bufs;
839 for (n = 0; n < tx_info->num_of_bufs; n++) {
840 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
841 nm_info->map_seg[n]);
842 }
843 tx_info->num_of_bufs = 0;
844
845 /* Next, retain the sockets back to the userspace */
846 for (n = 0; n < nm_info->sockets_used; n++) {
847 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
848 ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0,
849 ctx->adapter->ena_dev, "Tx idx is not 0.\n");
850 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
851 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
852 nm_info->socket_buf_idx[n] = 0;
853 }
854 nm_info->sockets_used = 0;
855
856 ctx->ring->free_tx_ids[ctx->nt] = req_id;
857 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->lim);
858
859 return tx_info->tx_descs;
860 }
861
862 static inline int
validate_tx_req_id(struct ena_ring * tx_ring,uint16_t req_id)863 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
864 {
865 struct ena_adapter *adapter = tx_ring->adapter;
866
867 if (likely(req_id < tx_ring->ring_size))
868 return (0);
869
870 ena_log_nm(adapter->pdev, WARN, "Invalid req_id %hu in qid %hu\n",
871 req_id, tx_ring->qid);
872 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
873
874 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
875
876 return (EFAULT);
877 }
878
879 static int
ena_netmap_rxsync(struct netmap_kring * kring,int flags)880 ena_netmap_rxsync(struct netmap_kring *kring, int flags)
881 {
882 struct ena_netmap_ctx ctx;
883 int rc;
884
885 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id));
886 ctx.ring = &ctx.adapter->rx_ring[kring->ring_id];
887
888 if (ctx.kring->rhead > ctx.lim) {
889 /* Probably not needed to release slots from RX ring. */
890 return (netmap_ring_reinit(ctx.kring));
891 }
892
893 if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0))
894 return (0);
895
896 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
897 return (0);
898
899 if ((rc = ena_netmap_rx_frames(&ctx)) != 0)
900 return (rc);
901
902 ena_netmap_rx_cleanup(&ctx);
903
904 return (0);
905 }
906
907 static inline int
ena_netmap_rx_frames(struct ena_netmap_ctx * ctx)908 ena_netmap_rx_frames(struct ena_netmap_ctx *ctx)
909 {
910 int rc = 0;
911 int frames_counter = 0;
912
913 ctx->nt = ctx->ring->next_to_clean;
914 ctx->nm_i = ctx->kring->nr_hwtail;
915
916 while ((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) {
917 frames_counter++;
918 /* In case of multiple frames, it is not an error. */
919 rc = 0;
920 if (frames_counter > ENA_MAX_FRAMES) {
921 ena_log_nm(ctx->adapter->pdev, ERR,
922 "Driver is stuck in the Rx loop\n");
923 break;
924 }
925 };
926
927 ctx->kring->nr_hwtail = ctx->nm_i;
928 ctx->kring->nr_kflags &= ~NKR_PENDINTR;
929 ctx->ring->next_to_clean = ctx->nt;
930
931 return (rc);
932 }
933
934 static inline int
ena_netmap_rx_frame(struct ena_netmap_ctx * ctx)935 ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
936 {
937 struct ena_com_rx_ctx ena_rx_ctx;
938 enum ena_regs_reset_reason_types reset_reason;
939 int rc, len = 0;
940 uint16_t buf, nm;
941
942 ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
943 ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
944 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
945 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
946
947 rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
948 if (unlikely(rc != 0)) {
949 ena_log_nm(ctx->adapter->pdev, ERR,
950 "Failed to read pkt from the device with error: %d\n", rc);
951 if (rc == ENA_COM_NO_SPACE) {
952 counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
953 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
954 } else {
955 counter_u64_add(ctx->ring->rx_stats.bad_req_id, 1);
956 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
957 }
958 ena_trigger_reset(ctx->adapter, reset_reason);
959 return (rc);
960 }
961 if (unlikely(ena_rx_ctx.descs == 0))
962 return (ENA_NETMAP_NO_MORE_FRAMES);
963
964 ena_log_nm(ctx->adapter->pdev, DBG,
965 "Rx: q %d got packet from ena. descs #:"
966 " %d l3 proto %d l4 proto %d hash: %x\n",
967 ctx->ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
968 ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
969
970 for (buf = 0; buf < ena_rx_ctx.descs; buf++)
971 if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
972 break;
973 /*
974 * ena_netmap_rx_load_desc doesn't know the number of descriptors.
975 * It just set flag NS_MOREFRAG to all slots, then here flag of
976 * last slot is cleared.
977 */
978 ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED;
979
980 if (rc != 0) {
981 goto rx_clear_desc;
982 }
983
984 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
985 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
986
987 counter_enter();
988 counter_u64_add_protected(ctx->ring->rx_stats.bytes, len);
989 counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len);
990 counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1);
991 counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1);
992 counter_exit();
993
994 return (ENA_NETMAP_MORE_FRAMES);
995
996 rx_clear_desc:
997 nm = ctx->nm_i;
998
999 /* Remove failed packet from ring */
1000 while (buf--) {
1001 ctx->slots[nm].flags = 0;
1002 ctx->slots[nm].len = 0;
1003 nm = nm_prev(nm, ctx->lim);
1004 }
1005
1006 return (rc);
1007 }
1008
1009 static inline int
ena_netmap_rx_load_desc(struct ena_netmap_ctx * ctx,uint16_t buf,int * len)1010 ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
1011 {
1012 struct ena_rx_buffer *rx_info;
1013 uint16_t req_id;
1014
1015 req_id = ctx->ring->ena_bufs[buf].req_id;
1016 rx_info = &ctx->ring->rx_buffer_info[req_id];
1017 bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
1018 BUS_DMASYNC_POSTREAD);
1019 netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
1020
1021 ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0, ctx->adapter->ena_dev,
1022 "Rx idx is not 0.\n");
1023
1024 ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
1025 rx_info->netmap_buf_idx = 0;
1026 /*
1027 * Set NS_MOREFRAG to all slots.
1028 * Then ena_netmap_rx_frame clears it from last one.
1029 */
1030 ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED;
1031 ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
1032 *len += ctx->slots[ctx->nm_i].len;
1033 ctx->ring->free_rx_ids[ctx->nt] = req_id;
1034 ena_log_nm(ctx->adapter->pdev, DBG,
1035 "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n", rx_info,
1036 ctx->slots[ctx->nm_i].buf_idx, (uintmax_t)rx_info->ena_buf.paddr,
1037 ctx->nm_i);
1038
1039 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
1040 ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
1041
1042 return (0);
1043 }
1044
1045 static inline void
ena_netmap_rx_cleanup(struct ena_netmap_ctx * ctx)1046 ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx)
1047 {
1048 int refill_required;
1049
1050 refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur;
1051 if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail)
1052 refill_required -= 1;
1053
1054 if (refill_required == 0)
1055 return;
1056 else if (refill_required < 0)
1057 refill_required += ctx->kring->nkr_num_slots;
1058
1059 ena_refill_rx_bufs(ctx->ring, refill_required);
1060 }
1061
1062 static inline void
ena_netmap_fill_ctx(struct netmap_kring * kring,struct ena_netmap_ctx * ctx,uint16_t ena_qid)1063 ena_netmap_fill_ctx(struct netmap_kring *kring, struct ena_netmap_ctx *ctx,
1064 uint16_t ena_qid)
1065 {
1066 ctx->kring = kring;
1067 ctx->na = kring->na;
1068 ctx->adapter = ctx->na->ifp->if_softc;
1069 ctx->lim = kring->nkr_num_slots - 1;
1070 ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid];
1071 ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid];
1072 ctx->slots = kring->ring->slot;
1073 }
1074
1075 void
ena_netmap_unload(struct ena_adapter * adapter,bus_dmamap_t map)1076 ena_netmap_unload(struct ena_adapter *adapter, bus_dmamap_t map)
1077 {
1078 struct netmap_adapter *na = NA(adapter->ifp);
1079
1080 netmap_unload_map(na, adapter->tx_buf_tag, map);
1081 }
1082
1083 #endif /* DEV_NETMAP */
1084