1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 #include <sys/cdefs.h>
31 #include "opt_rss.h"
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/eventhandler.h>
38 #include <sys/kernel.h>
39 #include <sys/kthread.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/rman.h>
44 #include <sys/smp.h>
45 #include <sys/socket.h>
46 #include <sys/sockio.h>
47 #include <sys/sysctl.h>
48 #include <sys/taskqueue.h>
49 #include <sys/time.h>
50
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53
54 #include <machine/atomic.h>
55 #include <machine/bus.h>
56 #include <machine/in_cksum.h>
57 #include <machine/resource.h>
58
59 #include <dev/pci/pcireg.h>
60 #include <dev/pci/pcivar.h>
61
62 #include <net/bpf.h>
63 #include <net/ethernet.h>
64 #include <net/if.h>
65 #include <net/if_arp.h>
66 #include <net/if_dl.h>
67 #include <net/if_media.h>
68 #include <net/if_types.h>
69 #include <net/if_var.h>
70 #include <net/if_vlan_var.h>
71 #include <netinet/in.h>
72 #include <netinet/in_systm.h>
73 #include <netinet/if_ether.h>
74 #include <netinet/ip.h>
75 #include <netinet/ip6.h>
76 #include <netinet/tcp.h>
77 #include <netinet/udp.h>
78
79 #include "ena.h"
80 #include "ena_datapath.h"
81 #include "ena_rss.h"
82 #include "ena_sysctl.h"
83
84 #ifdef DEV_NETMAP
85 #include "ena_netmap.h"
86 #endif /* DEV_NETMAP */
87
88 /*********************************************************
89 * Function prototypes
90 *********************************************************/
91 static int ena_probe(device_t);
92 static void ena_intr_msix_mgmnt(void *);
93 static void ena_free_pci_resources(struct ena_adapter *);
94 static int ena_change_mtu(if_t, int);
95 static inline void ena_alloc_counters(counter_u64_t *, int);
96 static inline void ena_free_counters(counter_u64_t *, int);
97 static inline void ena_reset_counters(counter_u64_t *, int);
98 static void ena_init_io_rings_common(struct ena_adapter *, struct ena_ring *,
99 uint16_t);
100 static void ena_init_io_rings_basic(struct ena_adapter *);
101 static void ena_init_io_rings_advanced(struct ena_adapter *);
102 static void ena_init_io_rings(struct ena_adapter *);
103 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
104 static void ena_free_all_io_rings_resources(struct ena_adapter *);
105 static int ena_setup_tx_dma_tag(struct ena_adapter *);
106 static int ena_free_tx_dma_tag(struct ena_adapter *);
107 static int ena_setup_rx_dma_tag(struct ena_adapter *);
108 static int ena_free_rx_dma_tag(struct ena_adapter *);
109 static void ena_release_all_tx_dmamap(struct ena_ring *);
110 static int ena_setup_tx_resources(struct ena_adapter *, int);
111 static void ena_free_tx_resources(struct ena_adapter *, int);
112 static int ena_setup_all_tx_resources(struct ena_adapter *);
113 static void ena_free_all_tx_resources(struct ena_adapter *);
114 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
115 static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
116 static int ena_setup_all_rx_resources(struct ena_adapter *);
117 static void ena_free_all_rx_resources(struct ena_adapter *);
118 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
119 struct ena_rx_buffer *);
120 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
121 struct ena_rx_buffer *);
122 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
123 static void ena_refill_all_rx_bufs(struct ena_adapter *);
124 static void ena_free_all_rx_bufs(struct ena_adapter *);
125 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int);
126 static void ena_free_all_tx_bufs(struct ena_adapter *);
127 static void ena_destroy_all_tx_queues(struct ena_adapter *);
128 static void ena_destroy_all_rx_queues(struct ena_adapter *);
129 static void ena_destroy_all_io_queues(struct ena_adapter *);
130 static int ena_create_io_queues(struct ena_adapter *);
131 static int ena_handle_msix(void *);
132 static int ena_enable_msix(struct ena_adapter *);
133 static void ena_setup_mgmnt_intr(struct ena_adapter *);
134 static int ena_setup_io_intr(struct ena_adapter *);
135 static int ena_request_mgmnt_irq(struct ena_adapter *);
136 static int ena_request_io_irq(struct ena_adapter *);
137 static void ena_free_mgmnt_irq(struct ena_adapter *);
138 static void ena_free_io_irq(struct ena_adapter *);
139 static void ena_free_irqs(struct ena_adapter *);
140 static void ena_disable_msix(struct ena_adapter *);
141 static void ena_unmask_all_io_irqs(struct ena_adapter *);
142 static int ena_up_complete(struct ena_adapter *);
143 static uint64_t ena_get_counter(if_t, ift_counter);
144 static int ena_media_change(if_t);
145 static void ena_media_status(if_t, struct ifmediareq *);
146 static void ena_init(void *);
147 static int ena_ioctl(if_t, u_long, caddr_t);
148 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
149 static void ena_update_host_info(struct ena_admin_host_info *, if_t);
150 static void ena_update_hwassist(struct ena_adapter *);
151 static void ena_setup_ifnet(device_t, struct ena_adapter *,
152 struct ena_com_dev_get_features_ctx *);
153 static int ena_enable_wc(device_t, struct resource *);
154 static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
155 struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
156 static int ena_map_llq_mem_bar(device_t, struct ena_com_dev *);
157 static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *,
158 struct ena_com_dev_get_features_ctx *);
159 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *, struct ena_adapter *);
160 static void ena_config_host_info(struct ena_com_dev *, device_t);
161 static int ena_attach(device_t);
162 static int ena_detach(device_t);
163 static int ena_device_init(struct ena_adapter *, device_t,
164 struct ena_com_dev_get_features_ctx *, int *);
165 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *);
166 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
167 static void unimplemented_aenq_handler(void *, struct ena_admin_aenq_entry *);
168 static int ena_copy_eni_metrics(struct ena_adapter *);
169 static int ena_copy_srd_metrics(struct ena_adapter *);
170 static int ena_copy_customer_metrics(struct ena_adapter *);
171 static void ena_timer_service(void *);
172 static enum ena_regs_reset_reason_types check_cdesc_in_tx_cq(struct ena_adapter *,
173 struct ena_ring *);
174 #ifdef DEV_NETMAP
175 static int ena_reinit_netmap(struct ena_adapter *adapter);
176 #endif
177
178 static char ena_version[] = ENA_DEVICE_NAME ENA_DRV_MODULE_NAME
179 " v" ENA_DRV_MODULE_VERSION;
180
181 static ena_vendor_info_t ena_vendor_info_array[] = {
182 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0 },
183 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0 },
184 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0 },
185 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0 },
186 /* Last entry */
187 { 0, 0, 0 }
188 };
189
190 struct sx ena_global_lock;
191
192 /*
193 * Contains pointers to event handlers, e.g. link state chage.
194 */
195 static struct ena_aenq_handlers aenq_handlers;
196
197 void
ena_dmamap_callback(void * arg,bus_dma_segment_t * segs,int nseg,int error)198 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
199 {
200 if (error != 0)
201 return;
202 *(bus_addr_t *)arg = segs[0].ds_addr;
203 }
204
205 int
ena_dma_alloc(device_t dmadev,bus_size_t size,ena_mem_handle_t * dma,int mapflags,bus_size_t alignment,int domain)206 ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
207 int mapflags, bus_size_t alignment, int domain)
208 {
209 struct ena_adapter *adapter = device_get_softc(dmadev);
210 device_t pdev = adapter->pdev;
211 uint32_t maxsize;
212 uint64_t dma_space_addr;
213 int error;
214
215 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
216
217 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
218 if (unlikely(dma_space_addr == 0))
219 dma_space_addr = BUS_SPACE_MAXADDR;
220
221 error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
222 alignment, 0, /* alignment, bounds */
223 dma_space_addr, /* lowaddr of exclusion window */
224 BUS_SPACE_MAXADDR, /* highaddr of exclusion window */
225 NULL, NULL, /* filter, filterarg */
226 maxsize, /* maxsize */
227 1, /* nsegments */
228 maxsize, /* maxsegsize */
229 BUS_DMA_ALLOCNOW, /* flags */
230 NULL, /* lockfunc */
231 NULL, /* lockarg */
232 &dma->tag);
233 if (unlikely(error != 0)) {
234 ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error);
235 goto fail_tag;
236 }
237
238 error = bus_dma_tag_set_domain(dma->tag, domain);
239 if (unlikely(error != 0)) {
240 ena_log(pdev, ERR, "bus_dma_tag_set_domain failed: %d\n",
241 error);
242 goto fail_map_create;
243 }
244
245 error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr,
246 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
247 if (unlikely(error != 0)) {
248 ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n",
249 (uintmax_t)size, error);
250 goto fail_map_create;
251 }
252
253 dma->paddr = 0;
254 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size,
255 ena_dmamap_callback, &dma->paddr, mapflags);
256 if (unlikely((error != 0) || (dma->paddr == 0))) {
257 ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error);
258 goto fail_map_load;
259 }
260
261 bus_dmamap_sync(dma->tag, dma->map,
262 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
263
264 return (0);
265
266 fail_map_load:
267 bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
268 fail_map_create:
269 bus_dma_tag_destroy(dma->tag);
270 fail_tag:
271 dma->tag = NULL;
272 dma->vaddr = NULL;
273 dma->paddr = 0;
274
275 return (error);
276 }
277
278 static void
ena_free_pci_resources(struct ena_adapter * adapter)279 ena_free_pci_resources(struct ena_adapter *adapter)
280 {
281 device_t pdev = adapter->pdev;
282
283 if (adapter->memory != NULL) {
284 bus_release_resource(pdev, SYS_RES_MEMORY,
285 PCIR_BAR(ENA_MEM_BAR), adapter->memory);
286 }
287
288 if (adapter->registers != NULL) {
289 bus_release_resource(pdev, SYS_RES_MEMORY,
290 PCIR_BAR(ENA_REG_BAR), adapter->registers);
291 }
292
293 if (adapter->msix != NULL) {
294 bus_release_resource(pdev, SYS_RES_MEMORY, adapter->msix_rid,
295 adapter->msix);
296 }
297 }
298
299 static int
ena_probe(device_t dev)300 ena_probe(device_t dev)
301 {
302 ena_vendor_info_t *ent;
303 uint16_t pci_vendor_id = 0;
304 uint16_t pci_device_id = 0;
305
306 pci_vendor_id = pci_get_vendor(dev);
307 pci_device_id = pci_get_device(dev);
308
309 ent = ena_vendor_info_array;
310 while (ent->vendor_id != 0) {
311 if ((pci_vendor_id == ent->vendor_id) &&
312 (pci_device_id == ent->device_id)) {
313 ena_log_raw(DBG, "vendor=%x device=%x\n", pci_vendor_id,
314 pci_device_id);
315
316 device_set_desc(dev, ENA_DEVICE_DESC);
317 return (BUS_PROBE_DEFAULT);
318 }
319
320 ent++;
321 }
322
323 return (ENXIO);
324 }
325
326 static int
ena_change_mtu(if_t ifp,int new_mtu)327 ena_change_mtu(if_t ifp, int new_mtu)
328 {
329 struct ena_adapter *adapter = if_getsoftc(ifp);
330 device_t pdev = adapter->pdev;
331 int rc;
332
333 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
334 ena_log(pdev, ERR, "Invalid MTU setting. new_mtu: %d max mtu: %d min mtu: %d\n",
335 new_mtu, adapter->max_mtu, ENA_MIN_MTU);
336 return (EINVAL);
337 }
338
339 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
340 if (likely(rc == 0)) {
341 ena_log(pdev, DBG, "set MTU to %d\n", new_mtu);
342 if_setmtu(ifp, new_mtu);
343 } else {
344 ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu);
345 }
346
347 return (rc);
348 }
349
350 static inline void
ena_alloc_counters(counter_u64_t * begin,int size)351 ena_alloc_counters(counter_u64_t *begin, int size)
352 {
353 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
354
355 for (; begin < end; ++begin)
356 *begin = counter_u64_alloc(M_WAITOK);
357 }
358
359 static inline void
ena_free_counters(counter_u64_t * begin,int size)360 ena_free_counters(counter_u64_t *begin, int size)
361 {
362 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
363
364 for (; begin < end; ++begin)
365 counter_u64_free(*begin);
366 }
367
368 static inline void
ena_reset_counters(counter_u64_t * begin,int size)369 ena_reset_counters(counter_u64_t *begin, int size)
370 {
371 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
372
373 for (; begin < end; ++begin)
374 counter_u64_zero(*begin);
375 }
376
377 static void
ena_init_io_rings_common(struct ena_adapter * adapter,struct ena_ring * ring,uint16_t qid)378 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
379 uint16_t qid)
380 {
381 ring->qid = qid;
382 ring->adapter = adapter;
383 ring->ena_dev = adapter->ena_dev;
384 atomic_store_8(&ring->first_interrupt, 0);
385 ring->no_interrupt_event_cnt = 0;
386 }
387
388 static void
ena_init_io_rings_basic(struct ena_adapter * adapter)389 ena_init_io_rings_basic(struct ena_adapter *adapter)
390 {
391 struct ena_com_dev *ena_dev;
392 struct ena_ring *txr, *rxr;
393 struct ena_que *que;
394 int i;
395
396 ena_dev = adapter->ena_dev;
397
398 for (i = 0; i < adapter->num_io_queues; i++) {
399 txr = &adapter->tx_ring[i];
400 rxr = &adapter->rx_ring[i];
401
402 /* TX/RX common ring state */
403 ena_init_io_rings_common(adapter, txr, i);
404 ena_init_io_rings_common(adapter, rxr, i);
405
406 /* TX specific ring state */
407 txr->tx_max_header_size = ena_dev->tx_max_header_size;
408 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
409
410 que = &adapter->que[i];
411 que->adapter = adapter;
412 que->id = i;
413 que->tx_ring = txr;
414 que->rx_ring = rxr;
415
416 txr->que = que;
417 rxr->que = que;
418
419 rxr->empty_rx_queue = 0;
420 rxr->rx_mbuf_sz = ena_mbuf_sz;
421 }
422 }
423
424 static void
ena_init_io_rings_advanced(struct ena_adapter * adapter)425 ena_init_io_rings_advanced(struct ena_adapter *adapter)
426 {
427 struct ena_ring *txr, *rxr;
428 int i;
429
430 for (i = 0; i < adapter->num_io_queues; i++) {
431 txr = &adapter->tx_ring[i];
432 rxr = &adapter->rx_ring[i];
433
434 /* Allocate a buf ring */
435 txr->buf_ring_size = adapter->buf_ring_size;
436 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, M_WAITOK,
437 &txr->ring_mtx);
438
439 /* Allocate Tx statistics. */
440 ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
441 sizeof(txr->tx_stats));
442 txr->tx_last_cleanup_ticks = ticks;
443
444 /* Allocate Rx statistics. */
445 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
446 sizeof(rxr->rx_stats));
447
448 /* Initialize locks */
449 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)",
450 device_get_nameunit(adapter->pdev), i);
451 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)",
452 device_get_nameunit(adapter->pdev), i);
453
454 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF);
455 }
456 }
457
458 static void
ena_init_io_rings(struct ena_adapter * adapter)459 ena_init_io_rings(struct ena_adapter *adapter)
460 {
461 /*
462 * IO rings initialization can be divided into the 2 steps:
463 * 1. Initialize variables and fields with initial values and copy
464 * them from adapter/ena_dev (basic)
465 * 2. Allocate mutex, counters and buf_ring (advanced)
466 */
467 ena_init_io_rings_basic(adapter);
468 ena_init_io_rings_advanced(adapter);
469 }
470
471 static void
ena_free_io_ring_resources(struct ena_adapter * adapter,unsigned int qid)472 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
473 {
474 struct ena_ring *txr = &adapter->tx_ring[qid];
475 struct ena_ring *rxr = &adapter->rx_ring[qid];
476
477 ena_free_counters((counter_u64_t *)&txr->tx_stats,
478 sizeof(txr->tx_stats));
479 ena_free_counters((counter_u64_t *)&rxr->rx_stats,
480 sizeof(rxr->rx_stats));
481
482 ENA_RING_MTX_LOCK(txr);
483 drbr_free(txr->br, M_DEVBUF);
484 ENA_RING_MTX_UNLOCK(txr);
485
486 mtx_destroy(&txr->ring_mtx);
487 }
488
489 static void
ena_free_all_io_rings_resources(struct ena_adapter * adapter)490 ena_free_all_io_rings_resources(struct ena_adapter *adapter)
491 {
492 int i;
493
494 for (i = 0; i < adapter->num_io_queues; i++)
495 ena_free_io_ring_resources(adapter, i);
496 }
497
498 static int
ena_setup_tx_dma_tag(struct ena_adapter * adapter)499 ena_setup_tx_dma_tag(struct ena_adapter *adapter)
500 {
501 int ret;
502
503 /* Create DMA tag for Tx buffers */
504 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
505 1, 0, /* alignment, bounds */
506 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
507 BUS_SPACE_MAXADDR, /* highaddr of excl window */
508 NULL, NULL, /* filter, filterarg */
509 ENA_TSO_MAXSIZE, /* maxsize */
510 adapter->max_tx_sgl_size - 1, /* nsegments */
511 ENA_TSO_MAXSIZE, /* maxsegsize */
512 0, /* flags */
513 NULL, /* lockfunc */
514 NULL, /* lockfuncarg */
515 &adapter->tx_buf_tag);
516
517 return (ret);
518 }
519
520 static int
ena_free_tx_dma_tag(struct ena_adapter * adapter)521 ena_free_tx_dma_tag(struct ena_adapter *adapter)
522 {
523 int ret;
524
525 ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
526
527 if (likely(ret == 0))
528 adapter->tx_buf_tag = NULL;
529
530 return (ret);
531 }
532
533 static int
ena_setup_rx_dma_tag(struct ena_adapter * adapter)534 ena_setup_rx_dma_tag(struct ena_adapter *adapter)
535 {
536 int ret;
537
538 /* Create DMA tag for Rx buffers*/
539 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */
540 1, 0, /* alignment, bounds */
541 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
542 BUS_SPACE_MAXADDR, /* highaddr of excl window */
543 NULL, NULL, /* filter, filterarg */
544 ena_mbuf_sz, /* maxsize */
545 adapter->max_rx_sgl_size, /* nsegments */
546 ena_mbuf_sz, /* maxsegsize */
547 0, /* flags */
548 NULL, /* lockfunc */
549 NULL, /* lockarg */
550 &adapter->rx_buf_tag);
551
552 return (ret);
553 }
554
555 static int
ena_free_rx_dma_tag(struct ena_adapter * adapter)556 ena_free_rx_dma_tag(struct ena_adapter *adapter)
557 {
558 int ret;
559
560 ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
561
562 if (likely(ret == 0))
563 adapter->rx_buf_tag = NULL;
564
565 return (ret);
566 }
567
568 int
validate_tx_req_id(struct ena_ring * tx_ring,uint16_t req_id,int tx_req_id_rc)569 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id, int tx_req_id_rc)
570 {
571 struct ena_adapter *adapter = tx_ring->adapter;
572 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
573
574 if (unlikely(tx_req_id_rc != 0)) {
575 if (tx_req_id_rc == ENA_COM_FAULT) {
576 reset_reason = ENA_REGS_RESET_TX_DESCRIPTOR_MALFORMED;
577 ena_log(adapter->pdev, ERR,
578 "TX descriptor malformed. req_id %hu qid %hu\n",
579 req_id, tx_ring->qid);
580 } else if (tx_req_id_rc == ENA_COM_INVAL) {
581 ena_log_nm(adapter->pdev, WARN,
582 "Invalid req_id %hu in qid %hu\n",
583 req_id, tx_ring->qid);
584 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
585 }
586
587 ena_trigger_reset(adapter, reset_reason);
588 return (EFAULT);
589 }
590
591 return (0);
592 }
593
594 static void
ena_release_all_tx_dmamap(struct ena_ring * tx_ring)595 ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
596 {
597 struct ena_adapter *adapter = tx_ring->adapter;
598 struct ena_tx_buffer *tx_info;
599 bus_dma_tag_t tx_tag = adapter->tx_buf_tag;
600 int i;
601 #ifdef DEV_NETMAP
602 struct ena_netmap_tx_info *nm_info;
603 int j;
604 #endif /* DEV_NETMAP */
605
606 for (i = 0; i < tx_ring->ring_size; ++i) {
607 tx_info = &tx_ring->tx_buffer_info[i];
608 #ifdef DEV_NETMAP
609 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
610 nm_info = &tx_info->nm_info;
611 for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
612 if (nm_info->map_seg[j] != NULL) {
613 bus_dmamap_destroy(tx_tag,
614 nm_info->map_seg[j]);
615 nm_info->map_seg[j] = NULL;
616 }
617 }
618 }
619 #endif /* DEV_NETMAP */
620 if (tx_info->dmamap != NULL) {
621 bus_dmamap_destroy(tx_tag, tx_info->dmamap);
622 tx_info->dmamap = NULL;
623 }
624 }
625 }
626
627 /**
628 * ena_setup_tx_resources - allocate Tx resources (Descriptors)
629 * @adapter: network interface device structure
630 * @qid: queue index
631 *
632 * Returns 0 on success, otherwise on failure.
633 **/
634 static int
ena_setup_tx_resources(struct ena_adapter * adapter,int qid)635 ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
636 {
637 device_t pdev = adapter->pdev;
638 char thread_name[MAXCOMLEN + 1];
639 struct ena_que *que = &adapter->que[qid];
640 struct ena_ring *tx_ring = que->tx_ring;
641 cpuset_t *cpu_mask = NULL;
642 int size, i, err;
643 #ifdef DEV_NETMAP
644 bus_dmamap_t *map;
645 int j;
646
647 ena_netmap_reset_tx_ring(adapter, qid);
648 #endif /* DEV_NETMAP */
649
650 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
651
652 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
653 if (unlikely(tx_ring->tx_buffer_info == NULL))
654 return (ENOMEM);
655
656 size = sizeof(uint16_t) * tx_ring->ring_size;
657 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
658 if (unlikely(tx_ring->free_tx_ids == NULL))
659 goto err_buf_info_free;
660
661 size = tx_ring->tx_max_header_size;
662 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF,
663 M_NOWAIT | M_ZERO);
664 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL))
665 goto err_tx_ids_free;
666
667 /* Req id stack for TX OOO completions */
668 for (i = 0; i < tx_ring->ring_size; i++)
669 tx_ring->free_tx_ids[i] = i;
670
671 /* Reset TX statistics. */
672 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
673 sizeof(tx_ring->tx_stats));
674
675 tx_ring->next_to_use = 0;
676 tx_ring->next_to_clean = 0;
677 tx_ring->acum_pkts = 0;
678
679 /* Make sure that drbr is empty */
680 ENA_RING_MTX_LOCK(tx_ring);
681 drbr_flush(adapter->ifp, tx_ring->br);
682 ENA_RING_MTX_UNLOCK(tx_ring);
683
684 /* ... and create the buffer DMA maps */
685 for (i = 0; i < tx_ring->ring_size; i++) {
686 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
687 &tx_ring->tx_buffer_info[i].dmamap);
688 if (unlikely(err != 0)) {
689 ena_log(pdev, ERR,
690 "Unable to create Tx DMA map for buffer %d\n", i);
691 goto err_map_release;
692 }
693
694 #ifdef DEV_NETMAP
695 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
696 map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
697 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
698 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
699 &map[j]);
700 if (unlikely(err != 0)) {
701 ena_log(pdev, ERR,
702 "Unable to create Tx DMA for buffer %d %d\n",
703 i, j);
704 goto err_map_release;
705 }
706 }
707 }
708 #endif /* DEV_NETMAP */
709 }
710
711 /* Allocate taskqueues */
712 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring);
713 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
714 taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
715 if (unlikely(tx_ring->enqueue_tq == NULL)) {
716 ena_log(pdev, ERR,
717 "Unable to create taskqueue for enqueue task\n");
718 i = tx_ring->ring_size;
719 goto err_map_release;
720 }
721
722 tx_ring->running = true;
723
724 #ifdef RSS
725 cpu_mask = &que->cpu_mask;
726 snprintf(thread_name, sizeof(thread_name), "%s txeq %d",
727 device_get_nameunit(adapter->pdev), que->cpu);
728 #else
729 snprintf(thread_name, sizeof(thread_name), "%s txeq %d",
730 device_get_nameunit(adapter->pdev), que->id);
731 #endif
732 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET,
733 cpu_mask, "%s", thread_name);
734
735 return (0);
736
737 err_map_release:
738 ena_release_all_tx_dmamap(tx_ring);
739 err_tx_ids_free:
740 free(tx_ring->free_tx_ids, M_DEVBUF);
741 tx_ring->free_tx_ids = NULL;
742 err_buf_info_free:
743 free(tx_ring->tx_buffer_info, M_DEVBUF);
744 tx_ring->tx_buffer_info = NULL;
745
746 return (ENOMEM);
747 }
748
749 /**
750 * ena_free_tx_resources - Free Tx Resources per Queue
751 * @adapter: network interface device structure
752 * @qid: queue index
753 *
754 * Free all transmit software resources
755 **/
756 static void
ena_free_tx_resources(struct ena_adapter * adapter,int qid)757 ena_free_tx_resources(struct ena_adapter *adapter, int qid)
758 {
759 struct ena_ring *tx_ring = &adapter->tx_ring[qid];
760 #ifdef DEV_NETMAP
761 struct ena_netmap_tx_info *nm_info;
762 int j;
763 #endif /* DEV_NETMAP */
764
765 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, NULL))
766 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
767
768 taskqueue_free(tx_ring->enqueue_tq);
769
770 ENA_RING_MTX_LOCK(tx_ring);
771 /* Flush buffer ring, */
772 drbr_flush(adapter->ifp, tx_ring->br);
773
774 /* Free buffer DMA maps, */
775 for (int i = 0; i < tx_ring->ring_size; i++) {
776 bus_dmamap_sync(adapter->tx_buf_tag,
777 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE);
778 bus_dmamap_unload(adapter->tx_buf_tag,
779 tx_ring->tx_buffer_info[i].dmamap);
780 bus_dmamap_destroy(adapter->tx_buf_tag,
781 tx_ring->tx_buffer_info[i].dmamap);
782
783 #ifdef DEV_NETMAP
784 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
785 nm_info = &tx_ring->tx_buffer_info[i].nm_info;
786 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
787 if (nm_info->socket_buf_idx[j] != 0) {
788 bus_dmamap_sync(adapter->tx_buf_tag,
789 nm_info->map_seg[j],
790 BUS_DMASYNC_POSTWRITE);
791 ena_netmap_unload(adapter,
792 nm_info->map_seg[j]);
793 }
794 bus_dmamap_destroy(adapter->tx_buf_tag,
795 nm_info->map_seg[j]);
796 nm_info->socket_buf_idx[j] = 0;
797 }
798 }
799 #endif /* DEV_NETMAP */
800
801 m_freem(tx_ring->tx_buffer_info[i].mbuf);
802 tx_ring->tx_buffer_info[i].mbuf = NULL;
803 }
804 ENA_RING_MTX_UNLOCK(tx_ring);
805
806 /* And free allocated memory. */
807 free(tx_ring->tx_buffer_info, M_DEVBUF);
808 tx_ring->tx_buffer_info = NULL;
809
810 free(tx_ring->free_tx_ids, M_DEVBUF);
811 tx_ring->free_tx_ids = NULL;
812
813 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
814 tx_ring->push_buf_intermediate_buf = NULL;
815 }
816
817 /**
818 * ena_setup_all_tx_resources - allocate all queues Tx resources
819 * @adapter: network interface device structure
820 *
821 * Returns 0 on success, otherwise on failure.
822 **/
823 static int
ena_setup_all_tx_resources(struct ena_adapter * adapter)824 ena_setup_all_tx_resources(struct ena_adapter *adapter)
825 {
826 int i, rc;
827
828 for (i = 0; i < adapter->num_io_queues; i++) {
829 rc = ena_setup_tx_resources(adapter, i);
830 if (rc != 0) {
831 ena_log(adapter->pdev, ERR,
832 "Allocation for Tx Queue %u failed\n", i);
833 goto err_setup_tx;
834 }
835 }
836
837 return (0);
838
839 err_setup_tx:
840 /* Rewind the index freeing the rings as we go */
841 while (i--)
842 ena_free_tx_resources(adapter, i);
843 return (rc);
844 }
845
846 /**
847 * ena_free_all_tx_resources - Free Tx Resources for All Queues
848 * @adapter: network interface device structure
849 *
850 * Free all transmit software resources
851 **/
852 static void
ena_free_all_tx_resources(struct ena_adapter * adapter)853 ena_free_all_tx_resources(struct ena_adapter *adapter)
854 {
855 int i;
856
857 for (i = 0; i < adapter->num_io_queues; i++)
858 ena_free_tx_resources(adapter, i);
859 }
860
861 /**
862 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
863 * @adapter: network interface device structure
864 * @qid: queue index
865 *
866 * Returns 0 on success, otherwise on failure.
867 **/
868 static int
ena_setup_rx_resources(struct ena_adapter * adapter,unsigned int qid)869 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
870 {
871 device_t pdev = adapter->pdev;
872 struct ena_que *que = &adapter->que[qid];
873 struct ena_ring *rx_ring = que->rx_ring;
874 int size, err, i;
875
876 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
877
878 #ifdef DEV_NETMAP
879 ena_netmap_reset_rx_ring(adapter, qid);
880 rx_ring->initialized = false;
881 #endif /* DEV_NETMAP */
882
883 /*
884 * Alloc extra element so in rx path
885 * we can always prefetch rx_info + 1
886 */
887 size += sizeof(struct ena_rx_buffer);
888
889 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
890
891 size = sizeof(uint16_t) * rx_ring->ring_size;
892 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);
893
894 for (i = 0; i < rx_ring->ring_size; i++)
895 rx_ring->free_rx_ids[i] = i;
896
897 /* Reset RX statistics. */
898 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
899 sizeof(rx_ring->rx_stats));
900
901 rx_ring->next_to_clean = 0;
902 rx_ring->next_to_use = 0;
903
904 /* ... and create the buffer DMA maps */
905 for (i = 0; i < rx_ring->ring_size; i++) {
906 err = bus_dmamap_create(adapter->rx_buf_tag, 0,
907 &(rx_ring->rx_buffer_info[i].map));
908 if (err != 0) {
909 ena_log(pdev, ERR,
910 "Unable to create Rx DMA map for buffer %d\n", i);
911 goto err_buf_info_unmap;
912 }
913 }
914
915 /* Create LRO for the ring */
916 if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) {
917 int err = tcp_lro_init(&rx_ring->lro);
918 if (err != 0) {
919 ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n",
920 qid);
921 } else {
922 ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n",
923 qid);
924 rx_ring->lro.ifp = adapter->ifp;
925 }
926 }
927
928 return (0);
929
930 err_buf_info_unmap:
931 while (i--) {
932 bus_dmamap_destroy(adapter->rx_buf_tag,
933 rx_ring->rx_buffer_info[i].map);
934 }
935
936 free(rx_ring->free_rx_ids, M_DEVBUF);
937 rx_ring->free_rx_ids = NULL;
938 free(rx_ring->rx_buffer_info, M_DEVBUF);
939 rx_ring->rx_buffer_info = NULL;
940 return (ENOMEM);
941 }
942
943 /**
944 * ena_free_rx_resources - Free Rx Resources
945 * @adapter: network interface device structure
946 * @qid: queue index
947 *
948 * Free all receive software resources
949 **/
950 static void
ena_free_rx_resources(struct ena_adapter * adapter,unsigned int qid)951 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid)
952 {
953 struct ena_ring *rx_ring = &adapter->rx_ring[qid];
954
955 /* Free buffer DMA maps, */
956 for (int i = 0; i < rx_ring->ring_size; i++) {
957 bus_dmamap_sync(adapter->rx_buf_tag,
958 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD);
959 m_freem(rx_ring->rx_buffer_info[i].mbuf);
960 rx_ring->rx_buffer_info[i].mbuf = NULL;
961 bus_dmamap_unload(adapter->rx_buf_tag,
962 rx_ring->rx_buffer_info[i].map);
963 bus_dmamap_destroy(adapter->rx_buf_tag,
964 rx_ring->rx_buffer_info[i].map);
965 }
966
967 /* free LRO resources, */
968 tcp_lro_free(&rx_ring->lro);
969
970 /* free allocated memory */
971 free(rx_ring->rx_buffer_info, M_DEVBUF);
972 rx_ring->rx_buffer_info = NULL;
973
974 free(rx_ring->free_rx_ids, M_DEVBUF);
975 rx_ring->free_rx_ids = NULL;
976 }
977
978 /**
979 * ena_setup_all_rx_resources - allocate all queues Rx resources
980 * @adapter: network interface device structure
981 *
982 * Returns 0 on success, otherwise on failure.
983 **/
984 static int
ena_setup_all_rx_resources(struct ena_adapter * adapter)985 ena_setup_all_rx_resources(struct ena_adapter *adapter)
986 {
987 int i, rc = 0;
988
989 for (i = 0; i < adapter->num_io_queues; i++) {
990 rc = ena_setup_rx_resources(adapter, i);
991 if (rc != 0) {
992 ena_log(adapter->pdev, ERR,
993 "Allocation for Rx Queue %u failed\n", i);
994 goto err_setup_rx;
995 }
996 }
997 return (0);
998
999 err_setup_rx:
1000 /* rewind the index freeing the rings as we go */
1001 while (i--)
1002 ena_free_rx_resources(adapter, i);
1003 return (rc);
1004 }
1005
1006 /**
1007 * ena_free_all_rx_resources - Free Rx resources for all queues
1008 * @adapter: network interface device structure
1009 *
1010 * Free all receive software resources
1011 **/
1012 static void
ena_free_all_rx_resources(struct ena_adapter * adapter)1013 ena_free_all_rx_resources(struct ena_adapter *adapter)
1014 {
1015 int i;
1016
1017 for (i = 0; i < adapter->num_io_queues; i++)
1018 ena_free_rx_resources(adapter, i);
1019 }
1020
1021 static inline int
ena_alloc_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)1022 ena_alloc_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
1023 struct ena_rx_buffer *rx_info)
1024 {
1025 device_t pdev = adapter->pdev;
1026 struct ena_com_buf *ena_buf;
1027 bus_dma_segment_t segs[1];
1028 int nsegs, error;
1029 int mlen;
1030
1031 /* if previous allocated frag is not used */
1032 if (unlikely(rx_info->mbuf != NULL))
1033 return (0);
1034
1035 /* Get mbuf using UMA allocator */
1036 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
1037 rx_ring->rx_mbuf_sz);
1038
1039 if (unlikely(rx_info->mbuf == NULL)) {
1040 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);
1041 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1042 if (unlikely(rx_info->mbuf == NULL)) {
1043 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
1044 return (ENOMEM);
1045 }
1046 mlen = MCLBYTES;
1047 } else {
1048 mlen = rx_ring->rx_mbuf_sz;
1049 }
1050 /* Set mbuf length*/
1051 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
1052
1053 /* Map packets for DMA */
1054 ena_log(pdev, DBG,
1055 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n",
1056 adapter->rx_buf_tag, rx_info->mbuf, rx_info->mbuf->m_len);
1057 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map,
1058 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
1059 if (unlikely((error != 0) || (nsegs != 1))) {
1060 ena_log(pdev, WARN,
1061 "failed to map mbuf, error: %d, nsegs: %d\n", error, nsegs);
1062 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
1063 goto exit;
1064 }
1065
1066 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
1067
1068 ena_buf = &rx_info->ena_buf;
1069 ena_buf->paddr = segs[0].ds_addr;
1070 ena_buf->len = mlen;
1071
1072 ena_log(pdev, DBG,
1073 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
1074 rx_info->mbuf, rx_info, ena_buf->len, (uintmax_t)ena_buf->paddr);
1075
1076 return (0);
1077
1078 exit:
1079 m_freem(rx_info->mbuf);
1080 rx_info->mbuf = NULL;
1081 return (EFAULT);
1082 }
1083
1084 static void
ena_free_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)1085 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
1086 struct ena_rx_buffer *rx_info)
1087 {
1088 if (rx_info->mbuf == NULL) {
1089 ena_log(adapter->pdev, WARN,
1090 "Trying to free unallocated buffer\n");
1091 return;
1092 }
1093
1094 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
1095 BUS_DMASYNC_POSTREAD);
1096 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
1097 m_freem(rx_info->mbuf);
1098 rx_info->mbuf = NULL;
1099 }
1100
1101 /**
1102 * ena_refill_rx_bufs - Refills ring with descriptors
1103 * @rx_ring: the ring which we want to feed with free descriptors
1104 * @num: number of descriptors to refill
1105 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1106 **/
1107 int
ena_refill_rx_bufs(struct ena_ring * rx_ring,uint32_t num)1108 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
1109 {
1110 struct ena_adapter *adapter = rx_ring->adapter;
1111 device_t pdev = adapter->pdev;
1112 uint16_t next_to_use, req_id;
1113 uint32_t i;
1114 int rc;
1115
1116 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid);
1117
1118 next_to_use = rx_ring->next_to_use;
1119
1120 for (i = 0; i < num; i++) {
1121 struct ena_rx_buffer *rx_info;
1122
1123 ena_log_io(pdev, DBG, "RX buffer - next to use: %d\n",
1124 next_to_use);
1125
1126 req_id = rx_ring->free_rx_ids[next_to_use];
1127 rx_info = &rx_ring->rx_buffer_info[req_id];
1128 #ifdef DEV_NETMAP
1129 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))
1130 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring,
1131 rx_info);
1132 else
1133 #endif /* DEV_NETMAP */
1134 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1135 if (unlikely(rc != 0)) {
1136 ena_log_io(pdev, WARN,
1137 "failed to alloc buffer for rx queue %d\n",
1138 rx_ring->qid);
1139 break;
1140 }
1141 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1142 &rx_info->ena_buf, req_id);
1143 if (unlikely(rc != 0)) {
1144 ena_log_io(pdev, WARN,
1145 "failed to add buffer for rx queue %d\n",
1146 rx_ring->qid);
1147 break;
1148 }
1149 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1150 rx_ring->ring_size);
1151 }
1152
1153 if (unlikely(i < num)) {
1154 counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
1155 ena_log_io(pdev, WARN,
1156 "refilled rx qid %d with only %d mbufs (from %d)\n",
1157 rx_ring->qid, i, num);
1158 }
1159
1160 if (likely(i != 0))
1161 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1162
1163 rx_ring->next_to_use = next_to_use;
1164 return (i);
1165 }
1166
1167 #ifdef DEV_NETMAP
1168 static int
ena_reinit_netmap(struct ena_adapter * adapter)1169 ena_reinit_netmap(struct ena_adapter *adapter)
1170 {
1171 int rc;
1172
1173 netmap_detach(adapter->ifp);
1174 rc = ena_netmap_attach(adapter);
1175 if (rc != 0)
1176 ena_log(adapter->pdev, ERR, "netmap attach failed: %d\n", rc);
1177
1178 return rc;
1179 }
1180
1181 #endif /* DEV_NETMAP */
1182 int
ena_update_buf_ring_size(struct ena_adapter * adapter,uint32_t new_buf_ring_size)1183 ena_update_buf_ring_size(struct ena_adapter *adapter,
1184 uint32_t new_buf_ring_size)
1185 {
1186 uint32_t old_buf_ring_size;
1187 int rc = 0;
1188 bool dev_was_up;
1189
1190 old_buf_ring_size = adapter->buf_ring_size;
1191 adapter->buf_ring_size = new_buf_ring_size;
1192
1193 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1194 ena_down(adapter);
1195
1196 /* Reconfigure buf ring for all Tx rings. */
1197 ena_free_all_io_rings_resources(adapter);
1198 ena_init_io_rings_advanced(adapter);
1199 #ifdef DEV_NETMAP
1200 rc = ena_reinit_netmap(adapter);
1201 if (rc != 0)
1202 return rc;
1203
1204 #endif /* DEV_NETMAP */
1205 if (dev_was_up) {
1206 /*
1207 * If ena_up() fails, it's not because of recent buf_ring size
1208 * changes. Because of that, we just want to revert old drbr
1209 * value and trigger the reset because something else had to
1210 * go wrong.
1211 */
1212 rc = ena_up(adapter);
1213 if (unlikely(rc != 0)) {
1214 ena_log(adapter->pdev, ERR,
1215 "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n",
1216 new_buf_ring_size, old_buf_ring_size);
1217
1218 /* Revert old size and trigger the reset */
1219 adapter->buf_ring_size = old_buf_ring_size;
1220 ena_free_all_io_rings_resources(adapter);
1221 ena_init_io_rings_advanced(adapter);
1222 #ifdef DEV_NETMAP
1223 rc = ena_reinit_netmap(adapter);
1224 if (rc != 0)
1225 return rc;
1226
1227 #endif /* DEV_NETMAP */
1228 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET,
1229 adapter);
1230 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER);
1231 }
1232 }
1233
1234 return (rc);
1235 }
1236
1237 int
ena_update_queue_size(struct ena_adapter * adapter,uint32_t new_tx_size,uint32_t new_rx_size)1238 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size,
1239 uint32_t new_rx_size)
1240 {
1241 uint32_t old_tx_size, old_rx_size;
1242 int rc = 0;
1243 bool dev_was_up;
1244
1245 old_tx_size = adapter->requested_tx_ring_size;
1246 old_rx_size = adapter->requested_rx_ring_size;
1247 adapter->requested_tx_ring_size = new_tx_size;
1248 adapter->requested_rx_ring_size = new_rx_size;
1249
1250 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1251 ena_down(adapter);
1252
1253 /* Configure queues with new size. */
1254 ena_init_io_rings_basic(adapter);
1255 #ifdef DEV_NETMAP
1256 rc = ena_reinit_netmap(adapter);
1257 if (rc != 0)
1258 return rc;
1259
1260 #endif /* DEV_NETMAP */
1261 if (dev_was_up) {
1262 rc = ena_up(adapter);
1263 if (unlikely(rc != 0)) {
1264 ena_log(adapter->pdev, ERR,
1265 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n",
1266 new_tx_size, new_rx_size, old_tx_size, old_rx_size);
1267
1268 /* Revert old size. */
1269 adapter->requested_tx_ring_size = old_tx_size;
1270 adapter->requested_rx_ring_size = old_rx_size;
1271 ena_init_io_rings_basic(adapter);
1272 #ifdef DEV_NETMAP
1273 rc = ena_reinit_netmap(adapter);
1274 if (rc != 0)
1275 return rc;
1276
1277 #endif /* DEV_NETMAP */
1278 /* And try again. */
1279 rc = ena_up(adapter);
1280 if (unlikely(rc != 0)) {
1281 ena_log(adapter->pdev, ERR,
1282 "Failed to revert old queue sizes. Triggering device reset.\n");
1283 /*
1284 * If we've failed again, something had to go
1285 * wrong. After reset, the device should try to
1286 * go up
1287 */
1288 ENA_FLAG_SET_ATOMIC(
1289 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1290 ena_trigger_reset(adapter,
1291 ENA_REGS_RESET_OS_TRIGGER);
1292 }
1293 }
1294 }
1295
1296 return (rc);
1297 }
1298
1299 static void
ena_update_io_rings(struct ena_adapter * adapter,uint32_t num)1300 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num)
1301 {
1302 ena_free_all_io_rings_resources(adapter);
1303 /* Force indirection table to be reinitialized */
1304 ena_com_rss_destroy(adapter->ena_dev);
1305
1306 adapter->num_io_queues = num;
1307 ena_init_io_rings(adapter);
1308 }
1309
1310 int
ena_update_base_cpu(struct ena_adapter * adapter,int new_num)1311 ena_update_base_cpu(struct ena_adapter *adapter, int new_num)
1312 {
1313 int old_num;
1314 int rc = 0;
1315 bool dev_was_up;
1316
1317 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1318 old_num = adapter->irq_cpu_base;
1319
1320 ena_down(adapter);
1321
1322 adapter->irq_cpu_base = new_num;
1323
1324 if (dev_was_up) {
1325 rc = ena_up(adapter);
1326 if (unlikely(rc != 0)) {
1327 ena_log(adapter->pdev, ERR,
1328 "Failed to configure device %d IRQ base CPU. "
1329 "Reverting to previous value: %d\n",
1330 new_num, old_num);
1331
1332 adapter->irq_cpu_base = old_num;
1333
1334 rc = ena_up(adapter);
1335 if (unlikely(rc != 0)) {
1336 ena_log(adapter->pdev, ERR,
1337 "Failed to revert to previous setup."
1338 "Triggering device reset.\n");
1339 ENA_FLAG_SET_ATOMIC(
1340 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1341 ena_trigger_reset(adapter,
1342 ENA_REGS_RESET_OS_TRIGGER);
1343 }
1344 }
1345 }
1346 return (rc);
1347 }
1348
1349 int
ena_update_cpu_stride(struct ena_adapter * adapter,uint32_t new_num)1350 ena_update_cpu_stride(struct ena_adapter *adapter, uint32_t new_num)
1351 {
1352 uint32_t old_num;
1353 int rc = 0;
1354 bool dev_was_up;
1355
1356 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1357 old_num = adapter->irq_cpu_stride;
1358
1359 ena_down(adapter);
1360
1361 adapter->irq_cpu_stride = new_num;
1362
1363 if (dev_was_up) {
1364 rc = ena_up(adapter);
1365 if (unlikely(rc != 0)) {
1366 ena_log(adapter->pdev, ERR,
1367 "Failed to configure device %d IRQ CPU stride. "
1368 "Reverting to previous value: %d\n",
1369 new_num, old_num);
1370
1371 adapter->irq_cpu_stride = old_num;
1372
1373 rc = ena_up(adapter);
1374 if (unlikely(rc != 0)) {
1375 ena_log(adapter->pdev, ERR,
1376 "Failed to revert to previous setup."
1377 "Triggering device reset.\n");
1378 ENA_FLAG_SET_ATOMIC(
1379 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1380 ena_trigger_reset(adapter,
1381 ENA_REGS_RESET_OS_TRIGGER);
1382 }
1383 }
1384 }
1385 return (rc);
1386 }
1387
1388 /* Caller should sanitize new_num */
1389 int
ena_update_io_queue_nb(struct ena_adapter * adapter,uint32_t new_num)1390 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num)
1391 {
1392 uint32_t old_num;
1393 int rc = 0;
1394 bool dev_was_up;
1395
1396 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1397 old_num = adapter->num_io_queues;
1398 ena_down(adapter);
1399
1400 ena_update_io_rings(adapter, new_num);
1401 #ifdef DEV_NETMAP
1402 rc = ena_reinit_netmap(adapter);
1403 if (rc != 0)
1404 return rc;
1405
1406 #endif /* DEV_NETMAP */
1407 if (dev_was_up) {
1408 rc = ena_up(adapter);
1409 if (unlikely(rc != 0)) {
1410 ena_log(adapter->pdev, ERR,
1411 "Failed to configure device with %u IO queues. "
1412 "Reverting to previous value: %u\n",
1413 new_num, old_num);
1414
1415 ena_update_io_rings(adapter, old_num);
1416 #ifdef DEV_NETMAP
1417 rc = ena_reinit_netmap(adapter);
1418 if (rc != 0)
1419 return rc;
1420
1421 #endif /* DEV_NETMAP */
1422 rc = ena_up(adapter);
1423 if (unlikely(rc != 0)) {
1424 ena_log(adapter->pdev, ERR,
1425 "Failed to revert to previous setup IO "
1426 "queues. Triggering device reset.\n");
1427 ENA_FLAG_SET_ATOMIC(
1428 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1429 ena_trigger_reset(adapter,
1430 ENA_REGS_RESET_OS_TRIGGER);
1431 }
1432 }
1433 }
1434
1435 return (rc);
1436 }
1437
1438 static void
ena_free_rx_bufs(struct ena_adapter * adapter,unsigned int qid)1439 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid)
1440 {
1441 struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1442 unsigned int i;
1443
1444 for (i = 0; i < rx_ring->ring_size; i++) {
1445 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1446
1447 if (rx_info->mbuf != NULL)
1448 ena_free_rx_mbuf(adapter, rx_ring, rx_info);
1449 #ifdef DEV_NETMAP
1450 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
1451 (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
1452 if (rx_info->netmap_buf_idx != 0)
1453 ena_netmap_free_rx_slot(adapter, rx_ring,
1454 rx_info);
1455 }
1456 #endif /* DEV_NETMAP */
1457 }
1458 }
1459
1460 /**
1461 * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1462 * @adapter: network interface device structure
1463 *
1464 */
1465 static void
ena_refill_all_rx_bufs(struct ena_adapter * adapter)1466 ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1467 {
1468 struct ena_ring *rx_ring;
1469 int i, rc, bufs_num;
1470
1471 for (i = 0; i < adapter->num_io_queues; i++) {
1472 rx_ring = &adapter->rx_ring[i];
1473 bufs_num = rx_ring->ring_size - 1;
1474 rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1475 if (unlikely(rc != bufs_num))
1476 ena_log_io(adapter->pdev, WARN,
1477 "refilling Queue %d failed. "
1478 "Allocated %d buffers from: %d\n",
1479 i, rc, bufs_num);
1480 #ifdef DEV_NETMAP
1481 rx_ring->initialized = true;
1482 #endif /* DEV_NETMAP */
1483 }
1484 }
1485
1486 static void
ena_free_all_rx_bufs(struct ena_adapter * adapter)1487 ena_free_all_rx_bufs(struct ena_adapter *adapter)
1488 {
1489 int i;
1490
1491 for (i = 0; i < adapter->num_io_queues; i++)
1492 ena_free_rx_bufs(adapter, i);
1493 }
1494
1495 /**
1496 * ena_free_tx_bufs - Free Tx Buffers per Queue
1497 * @adapter: network interface device structure
1498 * @qid: queue index
1499 **/
1500 static void
ena_free_tx_bufs(struct ena_adapter * adapter,unsigned int qid)1501 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
1502 {
1503 bool print_once = true;
1504 struct ena_ring *tx_ring = &adapter->tx_ring[qid];
1505
1506 ENA_RING_MTX_LOCK(tx_ring);
1507 for (int i = 0; i < tx_ring->ring_size; i++) {
1508 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1509
1510 if (tx_info->mbuf == NULL)
1511 continue;
1512
1513 if (print_once) {
1514 ena_log(adapter->pdev, WARN,
1515 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid,
1516 i);
1517 print_once = false;
1518 } else {
1519 ena_log(adapter->pdev, DBG,
1520 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid,
1521 i);
1522 }
1523
1524 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
1525 BUS_DMASYNC_POSTWRITE);
1526 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
1527
1528 m_free(tx_info->mbuf);
1529 tx_info->mbuf = NULL;
1530 }
1531 ENA_RING_MTX_UNLOCK(tx_ring);
1532 }
1533
1534 static void
ena_free_all_tx_bufs(struct ena_adapter * adapter)1535 ena_free_all_tx_bufs(struct ena_adapter *adapter)
1536 {
1537 for (int i = 0; i < adapter->num_io_queues; i++)
1538 ena_free_tx_bufs(adapter, i);
1539 }
1540
1541 static void
ena_destroy_all_tx_queues(struct ena_adapter * adapter)1542 ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1543 {
1544 uint16_t ena_qid;
1545 int i;
1546
1547 for (i = 0; i < adapter->num_io_queues; i++) {
1548 ena_qid = ENA_IO_TXQ_IDX(i);
1549 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1550 }
1551 }
1552
1553 static void
ena_destroy_all_rx_queues(struct ena_adapter * adapter)1554 ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1555 {
1556 uint16_t ena_qid;
1557 int i;
1558
1559 for (i = 0; i < adapter->num_io_queues; i++) {
1560 ena_qid = ENA_IO_RXQ_IDX(i);
1561 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1562 }
1563 }
1564
1565 static void
ena_destroy_all_io_queues(struct ena_adapter * adapter)1566 ena_destroy_all_io_queues(struct ena_adapter *adapter)
1567 {
1568 struct ena_que *queue;
1569 int i;
1570
1571 for (i = 0; i < adapter->num_io_queues; i++) {
1572 queue = &adapter->que[i];
1573 while (taskqueue_cancel(queue->cleanup_tq, &queue->cleanup_task, NULL))
1574 taskqueue_drain(queue->cleanup_tq, &queue->cleanup_task);
1575 taskqueue_free(queue->cleanup_tq);
1576 }
1577
1578 ena_destroy_all_tx_queues(adapter);
1579 ena_destroy_all_rx_queues(adapter);
1580 }
1581
1582 static int
ena_create_io_queues(struct ena_adapter * adapter)1583 ena_create_io_queues(struct ena_adapter *adapter)
1584 {
1585 struct ena_com_dev *ena_dev = adapter->ena_dev;
1586 struct ena_com_create_io_ctx ctx;
1587 struct ena_ring *ring;
1588 struct ena_que *queue;
1589 uint16_t ena_qid;
1590 uint32_t msix_vector;
1591 cpuset_t *cpu_mask = NULL;
1592 int rc, i;
1593
1594 /* Create TX queues */
1595 for (i = 0; i < adapter->num_io_queues; i++) {
1596 msix_vector = ENA_IO_IRQ_IDX(i);
1597 ena_qid = ENA_IO_TXQ_IDX(i);
1598 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1599 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1600 ctx.queue_size = adapter->requested_tx_ring_size;
1601 ctx.msix_vector = msix_vector;
1602 ctx.qid = ena_qid;
1603 ctx.numa_node = adapter->que[i].domain;
1604
1605 rc = ena_com_create_io_queue(ena_dev, &ctx);
1606 if (rc != 0) {
1607 ena_log(adapter->pdev, ERR,
1608 "Failed to create io TX queue #%d rc: %d\n", i, rc);
1609 goto err_tx;
1610 }
1611 ring = &adapter->tx_ring[i];
1612 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1613 &ring->ena_com_io_sq, &ring->ena_com_io_cq);
1614 if (rc != 0) {
1615 ena_log(adapter->pdev, ERR,
1616 "Failed to get TX queue handlers. TX queue num"
1617 " %d rc: %d\n",
1618 i, rc);
1619 ena_com_destroy_io_queue(ena_dev, ena_qid);
1620 goto err_tx;
1621 }
1622
1623 if (ctx.numa_node >= 0) {
1624 ena_com_update_numa_node(ring->ena_com_io_cq,
1625 ctx.numa_node);
1626 }
1627 }
1628
1629 /* Create RX queues */
1630 for (i = 0; i < adapter->num_io_queues; i++) {
1631 msix_vector = ENA_IO_IRQ_IDX(i);
1632 ena_qid = ENA_IO_RXQ_IDX(i);
1633 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1634 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1635 ctx.queue_size = adapter->requested_rx_ring_size;
1636 ctx.msix_vector = msix_vector;
1637 ctx.qid = ena_qid;
1638 ctx.numa_node = adapter->que[i].domain;
1639
1640 rc = ena_com_create_io_queue(ena_dev, &ctx);
1641 if (unlikely(rc != 0)) {
1642 ena_log(adapter->pdev, ERR,
1643 "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1644 goto err_rx;
1645 }
1646
1647 ring = &adapter->rx_ring[i];
1648 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1649 &ring->ena_com_io_sq, &ring->ena_com_io_cq);
1650 if (unlikely(rc != 0)) {
1651 ena_log(adapter->pdev, ERR,
1652 "Failed to get RX queue handlers. RX queue num"
1653 " %d rc: %d\n",
1654 i, rc);
1655 ena_com_destroy_io_queue(ena_dev, ena_qid);
1656 goto err_rx;
1657 }
1658
1659 if (ctx.numa_node >= 0) {
1660 ena_com_update_numa_node(ring->ena_com_io_cq,
1661 ctx.numa_node);
1662 }
1663 }
1664
1665 for (i = 0; i < adapter->num_io_queues; i++) {
1666 queue = &adapter->que[i];
1667
1668 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue);
1669 queue->cleanup_tq = taskqueue_create_fast("ena cleanup",
1670 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq);
1671
1672 #ifdef RSS
1673 cpu_mask = &queue->cpu_mask;
1674 #endif
1675 taskqueue_start_threads_cpuset(&queue->cleanup_tq, 1, PI_NET,
1676 cpu_mask, "%s queue %d cleanup",
1677 device_get_nameunit(adapter->pdev), i);
1678 }
1679
1680 return (0);
1681
1682 err_rx:
1683 while (i--)
1684 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1685 i = adapter->num_io_queues;
1686 err_tx:
1687 while (i--)
1688 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1689
1690 return (ENXIO);
1691 }
1692
1693 /*********************************************************************
1694 *
1695 * MSIX & Interrupt Service routine
1696 *
1697 **********************************************************************/
1698
1699 /**
1700 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1701 * @arg: interrupt number
1702 **/
1703 static void
ena_intr_msix_mgmnt(void * arg)1704 ena_intr_msix_mgmnt(void *arg)
1705 {
1706 struct ena_adapter *adapter = (struct ena_adapter *)arg;
1707
1708 ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1709 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)))
1710 ena_com_aenq_intr_handler(adapter->ena_dev, arg);
1711 }
1712
1713 /**
1714 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1715 * @arg: queue
1716 **/
1717 static int
ena_handle_msix(void * arg)1718 ena_handle_msix(void *arg)
1719 {
1720 struct ena_que *queue = arg;
1721 struct ena_adapter *adapter = queue->adapter;
1722 if_t ifp = adapter->ifp;
1723
1724 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
1725 return (FILTER_STRAY);
1726
1727 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task);
1728
1729 return (FILTER_HANDLED);
1730 }
1731
1732 static int
ena_enable_msix(struct ena_adapter * adapter)1733 ena_enable_msix(struct ena_adapter *adapter)
1734 {
1735 device_t dev = adapter->pdev;
1736 int msix_vecs, msix_req;
1737 int i, rc = 0;
1738
1739 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
1740 ena_log(dev, ERR, "Error, MSI-X is already enabled\n");
1741 return (EINVAL);
1742 }
1743
1744 /* Reserved the max msix vectors we might need */
1745 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1746
1747 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry),
1748 M_DEVBUF, M_WAITOK | M_ZERO);
1749
1750 ena_log(dev, DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs);
1751
1752 for (i = 0; i < msix_vecs; i++) {
1753 adapter->msix_entries[i].entry = i;
1754 /* Vectors must start from 1 */
1755 adapter->msix_entries[i].vector = i + 1;
1756 }
1757
1758 msix_req = msix_vecs;
1759 rc = pci_alloc_msix(dev, &msix_vecs);
1760 if (unlikely(rc != 0)) {
1761 ena_log(dev, ERR, "Failed to enable MSIX, vectors %d rc %d\n",
1762 msix_vecs, rc);
1763
1764 rc = ENOSPC;
1765 goto err_msix_free;
1766 }
1767
1768 if (msix_vecs != msix_req) {
1769 if (msix_vecs == ENA_ADMIN_MSIX_VEC) {
1770 ena_log(dev, ERR,
1771 "Not enough number of MSI-x allocated: %d\n",
1772 msix_vecs);
1773 pci_release_msi(dev);
1774 rc = ENOSPC;
1775 goto err_msix_free;
1776 }
1777 ena_log(dev, ERR,
1778 "Enable only %d MSI-x (out of %d), reduce "
1779 "the number of queues\n",
1780 msix_vecs, msix_req);
1781 }
1782
1783 adapter->msix_vecs = msix_vecs;
1784 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
1785
1786 return (0);
1787
1788 err_msix_free:
1789 free(adapter->msix_entries, M_DEVBUF);
1790 adapter->msix_entries = NULL;
1791
1792 return (rc);
1793 }
1794
1795 static void
ena_setup_mgmnt_intr(struct ena_adapter * adapter)1796 ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1797 {
1798 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, ENA_IRQNAME_SIZE,
1799 "ena-mgmnt@pci:%s", device_get_nameunit(adapter->pdev));
1800 /*
1801 * Handler is NULL on purpose, it will be set
1802 * when mgmnt interrupt is acquired
1803 */
1804 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL;
1805 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1806 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1807 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
1808 }
1809
1810 static int
ena_setup_io_intr(struct ena_adapter * adapter)1811 ena_setup_io_intr(struct ena_adapter *adapter)
1812 {
1813 #ifdef RSS
1814 int num_buckets = rss_getnumbuckets();
1815 static int last_bind = 0;
1816 int cur_bind;
1817 int idx;
1818 #endif
1819 int irq_idx;
1820
1821 if (adapter->msix_entries == NULL)
1822 return (EINVAL);
1823
1824 #ifdef RSS
1825 if (adapter->first_bind < 0) {
1826 adapter->first_bind = last_bind;
1827 last_bind = (last_bind + adapter->num_io_queues) % num_buckets;
1828 }
1829 cur_bind = adapter->first_bind;
1830 #endif
1831
1832 for (int i = 0; i < adapter->num_io_queues; i++) {
1833 irq_idx = ENA_IO_IRQ_IDX(i);
1834
1835 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1836 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i);
1837 adapter->irq_tbl[irq_idx].handler = ena_handle_msix;
1838 adapter->irq_tbl[irq_idx].data = &adapter->que[i];
1839 adapter->irq_tbl[irq_idx].vector =
1840 adapter->msix_entries[irq_idx].vector;
1841 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n",
1842 adapter->msix_entries[irq_idx].vector);
1843
1844 if (adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) {
1845 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1846 (unsigned)(adapter->irq_cpu_base +
1847 i * adapter->irq_cpu_stride) % (unsigned)mp_ncpus;
1848 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask);
1849 }
1850
1851 #ifdef RSS
1852 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1853 rss_getcpu(cur_bind);
1854 cur_bind = (cur_bind + 1) % num_buckets;
1855 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask);
1856
1857 for (idx = 0; idx < MAXMEMDOM; ++idx) {
1858 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx]))
1859 break;
1860 }
1861 adapter->que[i].domain = idx;
1862 #else
1863 adapter->que[i].domain = -1;
1864 #endif
1865 }
1866
1867 return (0);
1868 }
1869
1870 static int
ena_request_mgmnt_irq(struct ena_adapter * adapter)1871 ena_request_mgmnt_irq(struct ena_adapter *adapter)
1872 {
1873 device_t pdev = adapter->pdev;
1874 struct ena_irq *irq;
1875 unsigned long flags;
1876 int rc, rcc;
1877
1878 flags = RF_ACTIVE | RF_SHAREABLE;
1879
1880 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1881 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1882 &irq->vector, flags);
1883
1884 if (unlikely(irq->res == NULL)) {
1885 ena_log(pdev, ERR, "could not allocate irq vector: %d\n",
1886 irq->vector);
1887 return (ENXIO);
1888 }
1889
1890 rc = bus_setup_intr(adapter->pdev, irq->res,
1891 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, irq->data,
1892 &irq->cookie);
1893 if (unlikely(rc != 0)) {
1894 ena_log(pdev, ERR,
1895 "failed to register interrupt handler for irq %ju: %d\n",
1896 rman_get_start(irq->res), rc);
1897 goto err_res_free;
1898 }
1899 irq->requested = true;
1900
1901 return (rc);
1902
1903 err_res_free:
1904 ena_log(pdev, INFO, "releasing resource for irq %d\n", irq->vector);
1905 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, irq->vector,
1906 irq->res);
1907 if (unlikely(rcc != 0))
1908 ena_log(pdev, ERR,
1909 "dev has no parent while releasing res for irq: %d\n",
1910 irq->vector);
1911 irq->res = NULL;
1912
1913 return (rc);
1914 }
1915
1916 static int
ena_request_io_irq(struct ena_adapter * adapter)1917 ena_request_io_irq(struct ena_adapter *adapter)
1918 {
1919 device_t pdev = adapter->pdev;
1920 struct ena_irq *irq;
1921 unsigned long flags = 0;
1922 int rc = 0, i, rcc;
1923
1924 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) {
1925 ena_log(pdev, ERR,
1926 "failed to request I/O IRQ: MSI-X is not enabled\n");
1927 return (EINVAL);
1928 } else {
1929 flags = RF_ACTIVE | RF_SHAREABLE;
1930 }
1931
1932 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1933 irq = &adapter->irq_tbl[i];
1934
1935 if (unlikely(irq->requested))
1936 continue;
1937
1938 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1939 &irq->vector, flags);
1940 if (unlikely(irq->res == NULL)) {
1941 rc = ENOMEM;
1942 ena_log(pdev, ERR,
1943 "could not allocate irq vector: %d\n", irq->vector);
1944 goto err;
1945 }
1946
1947 rc = bus_setup_intr(adapter->pdev, irq->res,
1948 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, irq->data,
1949 &irq->cookie);
1950 if (unlikely(rc != 0)) {
1951 ena_log(pdev, ERR,
1952 "failed to register interrupt handler for irq %ju: %d\n",
1953 rman_get_start(irq->res), rc);
1954 goto err;
1955 }
1956 irq->requested = true;
1957
1958 if (adapter->rss_enabled || adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) {
1959 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu);
1960 if (unlikely(rc != 0)) {
1961 ena_log(pdev, ERR,
1962 "failed to bind interrupt handler for irq %ju to cpu %d: %d\n",
1963 rman_get_start(irq->res), irq->cpu, rc);
1964 goto err;
1965 }
1966
1967 ena_log(pdev, INFO, "queue %d - cpu %d\n",
1968 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
1969 }
1970 }
1971 return (rc);
1972
1973 err:
1974
1975 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) {
1976 irq = &adapter->irq_tbl[i];
1977 rcc = 0;
1978
1979 /* Once we entered err: section and irq->requested is true we
1980 free both intr and resources */
1981 if (irq->requested) {
1982 rcc = bus_teardown_intr(adapter->pdev, irq->res,
1983 irq->cookie);
1984 if (unlikely(rcc != 0))
1985 ena_log(pdev, ERR,
1986 "could not release irq: %d, error: %d\n",
1987 irq->vector, rcc);
1988 }
1989
1990 /* If we entered err: section without irq->requested set we know
1991 it was bus_alloc_resource_any() that needs cleanup, provided
1992 res is not NULL. In case res is NULL no work in needed in
1993 this iteration */
1994 rcc = 0;
1995 if (irq->res != NULL) {
1996 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1997 irq->vector, irq->res);
1998 }
1999 if (unlikely(rcc != 0))
2000 ena_log(pdev, ERR,
2001 "dev has no parent while releasing res for irq: %d\n",
2002 irq->vector);
2003 irq->requested = false;
2004 irq->res = NULL;
2005 }
2006
2007 return (rc);
2008 }
2009
2010 static void
ena_free_mgmnt_irq(struct ena_adapter * adapter)2011 ena_free_mgmnt_irq(struct ena_adapter *adapter)
2012 {
2013 device_t pdev = adapter->pdev;
2014 struct ena_irq *irq;
2015 int rc;
2016
2017 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2018 if (irq->requested) {
2019 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector);
2020 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
2021 if (unlikely(rc != 0))
2022 ena_log(pdev, ERR, "failed to tear down irq: %d\n",
2023 irq->vector);
2024 irq->requested = 0;
2025 }
2026
2027 if (irq->res != NULL) {
2028 ena_log(pdev, DBG, "release resource irq: %d\n", irq->vector);
2029 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2030 irq->vector, irq->res);
2031 irq->res = NULL;
2032 if (unlikely(rc != 0))
2033 ena_log(pdev, ERR,
2034 "dev has no parent while releasing res for irq: %d\n",
2035 irq->vector);
2036 }
2037 }
2038
2039 static void
ena_free_io_irq(struct ena_adapter * adapter)2040 ena_free_io_irq(struct ena_adapter *adapter)
2041 {
2042 device_t pdev = adapter->pdev;
2043 struct ena_irq *irq;
2044 int rc;
2045
2046 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
2047 irq = &adapter->irq_tbl[i];
2048 if (irq->requested) {
2049 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector);
2050 rc = bus_teardown_intr(adapter->pdev, irq->res,
2051 irq->cookie);
2052 if (unlikely(rc != 0)) {
2053 ena_log(pdev, ERR,
2054 "failed to tear down irq: %d\n",
2055 irq->vector);
2056 }
2057 irq->requested = 0;
2058 }
2059
2060 if (irq->res != NULL) {
2061 ena_log(pdev, DBG, "release resource irq: %d\n",
2062 irq->vector);
2063 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2064 irq->vector, irq->res);
2065 irq->res = NULL;
2066 if (unlikely(rc != 0)) {
2067 ena_log(pdev, ERR,
2068 "dev has no parent while releasing res for irq: %d\n",
2069 irq->vector);
2070 }
2071 }
2072 }
2073 }
2074
2075 static void
ena_free_irqs(struct ena_adapter * adapter)2076 ena_free_irqs(struct ena_adapter *adapter)
2077 {
2078 ena_free_io_irq(adapter);
2079 ena_free_mgmnt_irq(adapter);
2080 ena_disable_msix(adapter);
2081 }
2082
2083 static void
ena_disable_msix(struct ena_adapter * adapter)2084 ena_disable_msix(struct ena_adapter *adapter)
2085 {
2086 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
2087 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
2088 pci_release_msi(adapter->pdev);
2089 }
2090
2091 adapter->msix_vecs = 0;
2092 free(adapter->msix_entries, M_DEVBUF);
2093 adapter->msix_entries = NULL;
2094 }
2095
2096 static void
ena_unmask_all_io_irqs(struct ena_adapter * adapter)2097 ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2098 {
2099 struct ena_com_io_cq *io_cq;
2100 struct ena_eth_io_intr_reg intr_reg;
2101 struct ena_ring *tx_ring;
2102 uint16_t ena_qid;
2103 int i;
2104
2105 /* Unmask interrupts for all queues */
2106 for (i = 0; i < adapter->num_io_queues; i++) {
2107 ena_qid = ENA_IO_TXQ_IDX(i);
2108 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
2109 ena_com_update_intr_reg(&intr_reg, 0, 0, true, false);
2110 tx_ring = &adapter->tx_ring[i];
2111 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
2112 ena_com_unmask_intr(io_cq, &intr_reg);
2113 }
2114 }
2115
2116 static int
ena_up_complete(struct ena_adapter * adapter)2117 ena_up_complete(struct ena_adapter *adapter)
2118 {
2119 int rc;
2120
2121 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
2122 rc = ena_rss_configure(adapter);
2123 if (rc != 0) {
2124 ena_log(adapter->pdev, ERR,
2125 "Failed to configure RSS\n");
2126 return (rc);
2127 }
2128 }
2129
2130 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu);
2131 if (unlikely(rc != 0))
2132 return (rc);
2133
2134 ena_refill_all_rx_bufs(adapter);
2135 ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
2136 sizeof(adapter->hw_stats));
2137
2138 return (0);
2139 }
2140
2141 static void
set_io_rings_size(struct ena_adapter * adapter,int new_tx_size,int new_rx_size)2142 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, int new_rx_size)
2143 {
2144 int i;
2145
2146 for (i = 0; i < adapter->num_io_queues; i++) {
2147 adapter->tx_ring[i].ring_size = new_tx_size;
2148 adapter->rx_ring[i].ring_size = new_rx_size;
2149 }
2150 }
2151
2152 static int
create_queues_with_size_backoff(struct ena_adapter * adapter)2153 create_queues_with_size_backoff(struct ena_adapter *adapter)
2154 {
2155 device_t pdev = adapter->pdev;
2156 int rc;
2157 uint32_t cur_rx_ring_size, cur_tx_ring_size;
2158 uint32_t new_rx_ring_size, new_tx_ring_size;
2159
2160 /*
2161 * Current queue sizes might be set to smaller than the requested
2162 * ones due to past queue allocation failures.
2163 */
2164 set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2165 adapter->requested_rx_ring_size);
2166
2167 while (1) {
2168 /* Allocate transmit descriptors */
2169 rc = ena_setup_all_tx_resources(adapter);
2170 if (unlikely(rc != 0)) {
2171 ena_log(pdev, ERR, "err_setup_tx\n");
2172 goto err_setup_tx;
2173 }
2174
2175 /* Allocate receive descriptors */
2176 rc = ena_setup_all_rx_resources(adapter);
2177 if (unlikely(rc != 0)) {
2178 ena_log(pdev, ERR, "err_setup_rx\n");
2179 goto err_setup_rx;
2180 }
2181
2182 /* Create IO queues for Rx & Tx */
2183 rc = ena_create_io_queues(adapter);
2184 if (unlikely(rc != 0)) {
2185 ena_log(pdev, ERR, "create IO queues failed\n");
2186 goto err_io_que;
2187 }
2188
2189 return (0);
2190
2191 err_io_que:
2192 ena_free_all_rx_resources(adapter);
2193 err_setup_rx:
2194 ena_free_all_tx_resources(adapter);
2195 err_setup_tx:
2196 /*
2197 * Lower the ring size if ENOMEM. Otherwise, return the
2198 * error straightaway.
2199 */
2200 if (unlikely(rc != ENOMEM)) {
2201 ena_log(pdev, ERR,
2202 "Queue creation failed with error code: %d\n", rc);
2203 return (rc);
2204 }
2205
2206 cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2207 cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2208
2209 ena_log(pdev, ERR,
2210 "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2211 cur_tx_ring_size, cur_rx_ring_size);
2212
2213 new_tx_ring_size = cur_tx_ring_size;
2214 new_rx_ring_size = cur_rx_ring_size;
2215
2216 /*
2217 * Decrease the size of a larger queue, or decrease both if they
2218 * are the same size.
2219 */
2220 if (cur_rx_ring_size <= cur_tx_ring_size)
2221 new_tx_ring_size = cur_tx_ring_size / 2;
2222 if (cur_rx_ring_size >= cur_tx_ring_size)
2223 new_rx_ring_size = cur_rx_ring_size / 2;
2224
2225 if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2226 new_rx_ring_size < ENA_MIN_RING_SIZE) {
2227 ena_log(pdev, ERR,
2228 "Queue creation failed with the smallest possible queue size"
2229 "of %d for both queues. Not retrying with smaller queues\n",
2230 ENA_MIN_RING_SIZE);
2231 return (rc);
2232 }
2233
2234 ena_log(pdev, INFO,
2235 "Retrying queue creation with sizes TX=%d, RX=%d\n",
2236 new_tx_ring_size, new_rx_ring_size);
2237
2238 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size);
2239 }
2240 }
2241
2242 int
ena_up(struct ena_adapter * adapter)2243 ena_up(struct ena_adapter *adapter)
2244 {
2245 int rc = 0;
2246
2247 ENA_LOCK_ASSERT();
2248
2249 if (unlikely(device_is_attached(adapter->pdev) == 0)) {
2250 ena_log(adapter->pdev, ERR, "device is not attached!\n");
2251 return (ENXIO);
2252 }
2253
2254 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2255 return (0);
2256
2257 ena_log(adapter->pdev, INFO, "device is going UP\n");
2258
2259 /* setup interrupts for IO queues */
2260 rc = ena_setup_io_intr(adapter);
2261 if (unlikely(rc != 0)) {
2262 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n");
2263 goto error;
2264 }
2265 rc = ena_request_io_irq(adapter);
2266 if (unlikely(rc != 0)) {
2267 ena_log(adapter->pdev, ERR, "err_req_irq\n");
2268 goto error;
2269 }
2270
2271 ena_log(adapter->pdev, INFO,
2272 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, LLQ is %s\n",
2273 adapter->num_io_queues,
2274 adapter->requested_rx_ring_size,
2275 adapter->requested_tx_ring_size,
2276 (adapter->ena_dev->tx_mem_queue_type ==
2277 ENA_ADMIN_PLACEMENT_POLICY_DEV) ? "ENABLED" : "DISABLED");
2278
2279 rc = create_queues_with_size_backoff(adapter);
2280 if (unlikely(rc != 0)) {
2281 ena_log(adapter->pdev, ERR,
2282 "error creating queues with size backoff\n");
2283 goto err_create_queues_with_backoff;
2284 }
2285
2286 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
2287 if_link_state_change(adapter->ifp, LINK_STATE_UP);
2288
2289 rc = ena_up_complete(adapter);
2290 if (unlikely(rc != 0))
2291 goto err_up_complete;
2292
2293 counter_u64_add(adapter->dev_stats.interface_up, 1);
2294
2295 ena_update_hwassist(adapter);
2296
2297 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
2298
2299 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2300
2301 ena_unmask_all_io_irqs(adapter);
2302
2303 return (0);
2304
2305 err_up_complete:
2306 ena_destroy_all_io_queues(adapter);
2307 ena_free_all_rx_resources(adapter);
2308 ena_free_all_tx_resources(adapter);
2309 err_create_queues_with_backoff:
2310 ena_free_io_irq(adapter);
2311 error:
2312 return (rc);
2313 }
2314
2315 static uint64_t
ena_get_counter(if_t ifp,ift_counter cnt)2316 ena_get_counter(if_t ifp, ift_counter cnt)
2317 {
2318 struct ena_adapter *adapter;
2319 struct ena_hw_stats *stats;
2320
2321 adapter = if_getsoftc(ifp);
2322 stats = &adapter->hw_stats;
2323
2324 switch (cnt) {
2325 case IFCOUNTER_IPACKETS:
2326 return (counter_u64_fetch(stats->rx_packets));
2327 case IFCOUNTER_OPACKETS:
2328 return (counter_u64_fetch(stats->tx_packets));
2329 case IFCOUNTER_IBYTES:
2330 return (counter_u64_fetch(stats->rx_bytes));
2331 case IFCOUNTER_OBYTES:
2332 return (counter_u64_fetch(stats->tx_bytes));
2333 case IFCOUNTER_IQDROPS:
2334 return (counter_u64_fetch(stats->rx_drops));
2335 case IFCOUNTER_OQDROPS:
2336 return (counter_u64_fetch(stats->tx_drops));
2337 default:
2338 return (if_get_counter_default(ifp, cnt));
2339 }
2340 }
2341
2342 static int
ena_media_change(if_t ifp)2343 ena_media_change(if_t ifp)
2344 {
2345 /* Media Change is not supported by firmware */
2346 return (0);
2347 }
2348
2349 static void
ena_media_status(if_t ifp,struct ifmediareq * ifmr)2350 ena_media_status(if_t ifp, struct ifmediareq *ifmr)
2351 {
2352 struct ena_adapter *adapter = if_getsoftc(ifp);
2353 ena_log(adapter->pdev, DBG, "Media status update\n");
2354
2355 ENA_LOCK_LOCK();
2356
2357 ifmr->ifm_status = IFM_AVALID;
2358 ifmr->ifm_active = IFM_ETHER;
2359
2360 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) {
2361 ENA_LOCK_UNLOCK();
2362 ena_log(adapter->pdev, INFO, "Link is down\n");
2363 return;
2364 }
2365
2366 ifmr->ifm_status |= IFM_ACTIVE;
2367 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX;
2368
2369 ENA_LOCK_UNLOCK();
2370 }
2371
2372 static void
ena_init(void * arg)2373 ena_init(void *arg)
2374 {
2375 struct ena_adapter *adapter = (struct ena_adapter *)arg;
2376
2377 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) {
2378 ENA_LOCK_LOCK();
2379 ena_up(adapter);
2380 ENA_LOCK_UNLOCK();
2381 }
2382 }
2383
2384 static int
ena_ioctl(if_t ifp,u_long command,caddr_t data)2385 ena_ioctl(if_t ifp, u_long command, caddr_t data)
2386 {
2387 struct ena_adapter *adapter;
2388 struct ifreq *ifr;
2389 int rc;
2390
2391 adapter = ifp->if_softc;
2392 ifr = (struct ifreq *)data;
2393
2394 /*
2395 * Acquiring lock to prevent from running up and down routines parallel.
2396 */
2397 rc = 0;
2398 switch (command) {
2399 case SIOCSIFMTU:
2400 if (ifp->if_mtu == ifr->ifr_mtu)
2401 break;
2402 ENA_LOCK_LOCK();
2403 ena_down(adapter);
2404
2405 ena_change_mtu(ifp, ifr->ifr_mtu);
2406
2407 rc = ena_up(adapter);
2408 ENA_LOCK_UNLOCK();
2409 break;
2410
2411 case SIOCSIFFLAGS:
2412 if ((ifp->if_flags & IFF_UP) != 0) {
2413 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
2414 if ((ifp->if_flags &
2415 (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
2416 ena_log(adapter->pdev, INFO,
2417 "ioctl promisc/allmulti\n");
2418 }
2419 } else {
2420 ENA_LOCK_LOCK();
2421 rc = ena_up(adapter);
2422 ENA_LOCK_UNLOCK();
2423 }
2424 } else {
2425 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
2426 ENA_LOCK_LOCK();
2427 ena_down(adapter);
2428 ENA_LOCK_UNLOCK();
2429 }
2430 }
2431 break;
2432
2433 case SIOCADDMULTI:
2434 case SIOCDELMULTI:
2435 break;
2436
2437 case SIOCSIFMEDIA:
2438 case SIOCGIFMEDIA:
2439 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
2440 break;
2441
2442 case SIOCSIFCAP:
2443 {
2444 int reinit = 0;
2445
2446 if (ifr->ifr_reqcap != ifp->if_capenable) {
2447 ifp->if_capenable = ifr->ifr_reqcap;
2448 reinit = 1;
2449 }
2450
2451 if ((reinit != 0) &&
2452 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) {
2453 ENA_LOCK_LOCK();
2454 ena_down(adapter);
2455 rc = ena_up(adapter);
2456 ENA_LOCK_UNLOCK();
2457 }
2458 }
2459
2460 break;
2461 default:
2462 rc = ether_ioctl(ifp, command, data);
2463 break;
2464 }
2465
2466 return (rc);
2467 }
2468
2469 static int
ena_get_dev_offloads(struct ena_com_dev_get_features_ctx * feat)2470 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat)
2471 {
2472 int caps = 0;
2473
2474 if ((feat->offload.tx &
2475 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2476 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK |
2477 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0)
2478 caps |= IFCAP_TXCSUM;
2479
2480 if ((feat->offload.tx &
2481 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK |
2482 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0)
2483 caps |= IFCAP_TXCSUM_IPV6;
2484
2485 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0)
2486 caps |= IFCAP_TSO4;
2487
2488 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0)
2489 caps |= IFCAP_TSO6;
2490
2491 if ((feat->offload.rx_supported &
2492 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK |
2493 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0)
2494 caps |= IFCAP_RXCSUM;
2495
2496 if ((feat->offload.rx_supported &
2497 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0)
2498 caps |= IFCAP_RXCSUM_IPV6;
2499
2500 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU;
2501
2502 return (caps);
2503 }
2504
2505 static void
ena_update_host_info(struct ena_admin_host_info * host_info,if_t ifp)2506 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp)
2507 {
2508 host_info->supported_network_features[0] = (uint32_t)if_getcapabilities(ifp);
2509 }
2510
2511 static void
ena_update_hwassist(struct ena_adapter * adapter)2512 ena_update_hwassist(struct ena_adapter *adapter)
2513 {
2514 if_t ifp = adapter->ifp;
2515 uint32_t feat = adapter->tx_offload_cap;
2516 int cap = if_getcapenable(ifp);
2517 int flags = 0;
2518
2519 if_clearhwassist(ifp);
2520
2521 if ((cap & IFCAP_TXCSUM) != 0) {
2522 if ((feat &
2523 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0)
2524 flags |= CSUM_IP;
2525 if ((feat &
2526 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2527 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0)
2528 flags |= CSUM_IP_UDP | CSUM_IP_TCP;
2529 }
2530
2531 if ((cap & IFCAP_TXCSUM_IPV6) != 0)
2532 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP;
2533
2534 if ((cap & IFCAP_TSO4) != 0)
2535 flags |= CSUM_IP_TSO;
2536
2537 if ((cap & IFCAP_TSO6) != 0)
2538 flags |= CSUM_IP6_TSO;
2539
2540 if_sethwassistbits(ifp, flags, 0);
2541 }
2542
2543 static void
ena_setup_ifnet(device_t pdev,struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * feat)2544 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
2545 struct ena_com_dev_get_features_ctx *feat)
2546 {
2547 if_t ifp;
2548 int caps = 0;
2549
2550 ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2551 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
2552 if_setdev(ifp, pdev);
2553 if_setsoftc(ifp, adapter);
2554
2555 if_setflags(ifp,
2556 IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH);
2557 if_setinitfn(ifp, ena_init);
2558 if_settransmitfn(ifp, ena_mq_start);
2559 if_setqflushfn(ifp, ena_qflush);
2560 if_setioctlfn(ifp, ena_ioctl);
2561 if_setgetcounterfn(ifp, ena_get_counter);
2562
2563 if_setsendqlen(ifp, adapter->requested_tx_ring_size);
2564 if_setsendqready(ifp);
2565 if_setmtu(ifp, ETHERMTU);
2566 if_setbaudrate(ifp, 0);
2567 /* Zeroize capabilities... */
2568 if_setcapabilities(ifp, 0);
2569 if_setcapenable(ifp, 0);
2570 /* check hardware support */
2571 caps = ena_get_dev_offloads(feat);
2572 /* ... and set them */
2573 if_setcapabilitiesbit(ifp, caps, 0);
2574
2575 /* TSO parameters */
2576 ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
2577 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2578 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1;
2579 ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2580
2581 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2582 if_setcapenable(ifp, if_getcapabilities(ifp));
2583
2584 /*
2585 * Specify the media types supported by this adapter and register
2586 * callbacks to update media and link information
2587 */
2588 ifmedia_init(&adapter->media, IFM_IMASK, ena_media_change,
2589 ena_media_status);
2590 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2591 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2592
2593 ether_ifattach(ifp, adapter->mac_addr);
2594 }
2595
2596 void
ena_down(struct ena_adapter * adapter)2597 ena_down(struct ena_adapter *adapter)
2598 {
2599 int rc;
2600
2601 ENA_LOCK_ASSERT();
2602
2603 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2604 return;
2605
2606 ena_log(adapter->pdev, INFO, "device is going DOWN\n");
2607
2608 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2609 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
2610
2611 ena_free_io_irq(adapter);
2612
2613 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) {
2614 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2615 if (unlikely(rc != 0))
2616 ena_log(adapter->pdev, ERR, "Device reset failed\n");
2617 }
2618
2619 ena_destroy_all_io_queues(adapter);
2620
2621 ena_free_all_tx_bufs(adapter);
2622 ena_free_all_rx_bufs(adapter);
2623 ena_free_all_tx_resources(adapter);
2624 ena_free_all_rx_resources(adapter);
2625
2626 counter_u64_add(adapter->dev_stats.interface_down, 1);
2627 }
2628
2629 static uint32_t
ena_calc_max_io_queue_num(device_t pdev,struct ena_com_dev * ena_dev,struct ena_com_dev_get_features_ctx * get_feat_ctx)2630 ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev,
2631 struct ena_com_dev_get_features_ctx *get_feat_ctx)
2632 {
2633 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
2634
2635 /* Regular queues capabilities */
2636 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2637 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2638 &get_feat_ctx->max_queue_ext.max_queue_ext;
2639 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num,
2640 max_queue_ext->max_rx_cq_num);
2641
2642 io_tx_sq_num = max_queue_ext->max_tx_sq_num;
2643 io_tx_cq_num = max_queue_ext->max_tx_cq_num;
2644 } else {
2645 struct ena_admin_queue_feature_desc *max_queues =
2646 &get_feat_ctx->max_queues;
2647 io_tx_sq_num = max_queues->max_sq_num;
2648 io_tx_cq_num = max_queues->max_cq_num;
2649 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num);
2650 }
2651
2652 /* In case of LLQ use the llq fields for the tx SQ/CQ */
2653 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2654 io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
2655
2656 max_num_io_queues = min_t(uint32_t, mp_ncpus, ENA_MAX_NUM_IO_QUEUES);
2657 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_rx_num);
2658 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_sq_num);
2659 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_cq_num);
2660 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */
2661 max_num_io_queues = min_t(uint32_t, max_num_io_queues,
2662 pci_msix_count(pdev) - 1);
2663 #ifdef RSS
2664 max_num_io_queues = min_t(uint32_t, max_num_io_queues,
2665 rss_getnumbuckets());
2666 #endif
2667
2668 return (max_num_io_queues);
2669 }
2670
2671 static int
ena_enable_wc(device_t pdev,struct resource * res)2672 ena_enable_wc(device_t pdev, struct resource *res)
2673 {
2674 #if defined(__i386) || defined(__amd64) || defined(__aarch64__)
2675 vm_offset_t va;
2676 vm_size_t len;
2677 int rc;
2678
2679 va = (vm_offset_t)rman_get_virtual(res);
2680 len = rman_get_size(res);
2681 /* Enable write combining */
2682 rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING);
2683 if (unlikely(rc != 0)) {
2684 ena_log(pdev, ERR, "pmap_change_attr failed, %d\n", rc);
2685 return (rc);
2686 }
2687
2688 return (0);
2689 #endif
2690 return (EOPNOTSUPP);
2691 }
2692
2693 static int
ena_set_queues_placement_policy(device_t pdev,struct ena_com_dev * ena_dev,struct ena_admin_feature_llq_desc * llq,struct ena_llq_configurations * llq_default_configurations)2694 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev,
2695 struct ena_admin_feature_llq_desc *llq,
2696 struct ena_llq_configurations *llq_default_configurations)
2697 {
2698 int rc;
2699 uint32_t llq_feature_mask;
2700
2701 llq_feature_mask = 1 << ENA_ADMIN_LLQ;
2702 if (!(ena_dev->supported_features & llq_feature_mask)) {
2703 ena_log(pdev, WARN,
2704 "LLQ is not supported. Fallback to host mode policy.\n");
2705 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2706 return (0);
2707 }
2708
2709 if (ena_dev->mem_bar == NULL) {
2710 ena_log(pdev, WARN,
2711 "LLQ is advertised as supported but device doesn't expose mem bar.\n");
2712 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2713 return (0);
2714 }
2715
2716 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
2717 if (unlikely(rc != 0)) {
2718 ena_log(pdev, WARN,
2719 "Failed to configure the device mode. "
2720 "Fallback to host mode policy.\n");
2721 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2722 }
2723
2724 return (0);
2725 }
2726
2727 static int
ena_map_llq_mem_bar(device_t pdev,struct ena_com_dev * ena_dev)2728 ena_map_llq_mem_bar(device_t pdev, struct ena_com_dev *ena_dev)
2729 {
2730 struct ena_adapter *adapter = device_get_softc(pdev);
2731 int rc, rid;
2732
2733 /* Try to allocate resources for LLQ bar */
2734 rid = PCIR_BAR(ENA_MEM_BAR);
2735 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid,
2736 RF_ACTIVE);
2737 if (unlikely(adapter->memory == NULL)) {
2738 ena_log(pdev, WARN,
2739 "Unable to allocate LLQ bar resource. LLQ mode won't be used.\n");
2740 return (0);
2741 }
2742
2743 /* Enable write combining for better LLQ performance */
2744 rc = ena_enable_wc(adapter->pdev, adapter->memory);
2745 if (unlikely(rc != 0)) {
2746 ena_log(pdev, ERR, "failed to enable write combining.\n");
2747 return (rc);
2748 }
2749
2750 /*
2751 * Save virtual address of the device's memory region
2752 * for the ena_com layer.
2753 */
2754 ena_dev->mem_bar = rman_get_virtual(adapter->memory);
2755
2756 return (0);
2757 }
2758
2759 static inline void
ena_set_llq_configurations(struct ena_llq_configurations * llq_config,struct ena_admin_feature_llq_desc * llq,struct ena_adapter * adapter)2760 ena_set_llq_configurations(struct ena_llq_configurations *llq_config,
2761 struct ena_admin_feature_llq_desc *llq, struct ena_adapter *adapter)
2762 {
2763 bool use_large_llq;
2764
2765 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
2766 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
2767 llq_config->llq_num_decs_before_header =
2768 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
2769
2770 switch (ena_force_large_llq_header)
2771 {
2772 case ENA_LLQ_HEADER_SIZE_POLICY_REGULAR:
2773 use_large_llq = false;
2774 break;
2775 case ENA_LLQ_HEADER_SIZE_POLICY_LARGE:
2776 use_large_llq = true;
2777 break;
2778 case ENA_LLQ_HEADER_SIZE_POLICY_DEFAULT:
2779 use_large_llq =
2780 (llq->entry_size_recommended == ENA_ADMIN_LIST_ENTRY_SIZE_256B);
2781 break;
2782 default:
2783 use_large_llq = false;
2784 ena_log(adapter->pdev, WARN,
2785 "force_large_llq_header should have values [0-2]\n");
2786 break;
2787 }
2788
2789 if (!(llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B))
2790 use_large_llq = false;
2791
2792 if (use_large_llq) {
2793 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
2794 llq_config->llq_ring_entry_size_value = 256;
2795 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
2796 } else {
2797 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2798 llq_config->llq_ring_entry_size_value = 128;
2799 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2800 }
2801 }
2802
2803 static int
ena_calc_io_queue_size(struct ena_calc_queue_size_ctx * ctx,struct ena_adapter * adapter)2804 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, struct ena_adapter *adapter)
2805 {
2806 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
2807 struct ena_com_dev *ena_dev = ctx->ena_dev;
2808 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE;
2809 uint32_t rx_queue_size = ENA_DEFAULT_RING_SIZE;
2810 uint32_t max_tx_queue_size;
2811 uint32_t max_rx_queue_size;
2812
2813 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2814 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2815 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
2816 max_rx_queue_size = min_t(uint32_t,
2817 max_queue_ext->max_rx_cq_depth,
2818 max_queue_ext->max_rx_sq_depth);
2819 max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
2820
2821 if (ena_dev->tx_mem_queue_type ==
2822 ENA_ADMIN_PLACEMENT_POLICY_DEV)
2823 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2824 llq->max_llq_depth);
2825 else
2826 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2827 max_queue_ext->max_tx_sq_depth);
2828
2829 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2830 max_queue_ext->max_per_packet_tx_descs);
2831 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2832 max_queue_ext->max_per_packet_rx_descs);
2833 } else {
2834 struct ena_admin_queue_feature_desc *max_queues =
2835 &ctx->get_feat_ctx->max_queues;
2836 max_rx_queue_size = min_t(uint32_t, max_queues->max_cq_depth,
2837 max_queues->max_sq_depth);
2838 max_tx_queue_size = max_queues->max_cq_depth;
2839
2840 if (ena_dev->tx_mem_queue_type ==
2841 ENA_ADMIN_PLACEMENT_POLICY_DEV)
2842 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2843 llq->max_llq_depth);
2844 else
2845 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2846 max_queues->max_sq_depth);
2847
2848 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2849 max_queues->max_packet_tx_descs);
2850 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2851 max_queues->max_packet_rx_descs);
2852 }
2853
2854 if (adapter->llq_policy == ENA_ADMIN_LIST_ENTRY_SIZE_256B) {
2855 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2856 if (llq->max_wide_llq_depth != max_tx_queue_size) {
2857 if (llq->max_wide_llq_depth == 0) {
2858 /* if there is no large llq max depth from device, we divide
2859 * the queue size by 2, leaving the amount of memory
2860 * used by the queues unchanged.
2861 */
2862 max_tx_queue_size /= 2;
2863 } else {
2864 max_tx_queue_size = llq->max_wide_llq_depth;
2865 }
2866 ena_log(ctx->pdev, INFO,
2867 "Using large LLQ headers and decreasing maximum Tx queue size to %d\n",
2868 max_tx_queue_size);
2869 } else {
2870 ena_log(ctx->pdev, INFO, "Using large LLQ headers\n");
2871 }
2872 } else {
2873 ena_log(ctx->pdev, WARN,
2874 "Using large headers failed: LLQ is disabled or device does not support large headers\n");
2875 }
2876 }
2877
2878 /* round down to the nearest power of 2 */
2879 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1);
2880 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1);
2881
2882 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
2883 max_tx_queue_size);
2884 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
2885 max_rx_queue_size);
2886
2887 tx_queue_size = 1 << (flsl(tx_queue_size) - 1);
2888 rx_queue_size = 1 << (flsl(rx_queue_size) - 1);
2889
2890 ctx->max_tx_queue_size = max_tx_queue_size;
2891 ctx->max_rx_queue_size = max_rx_queue_size;
2892 ctx->tx_queue_size = tx_queue_size;
2893 ctx->rx_queue_size = rx_queue_size;
2894
2895 return (0);
2896 }
2897
2898 static void
ena_config_host_info(struct ena_com_dev * ena_dev,device_t dev)2899 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev)
2900 {
2901 struct ena_admin_host_info *host_info;
2902 uintptr_t rid;
2903 int rc;
2904
2905 /* Allocate only the host info */
2906 rc = ena_com_allocate_host_info(ena_dev);
2907 if (unlikely(rc != 0)) {
2908 ena_log(dev, ERR, "Cannot allocate host info\n");
2909 return;
2910 }
2911
2912 host_info = ena_dev->host_attr.host_info;
2913
2914 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0)
2915 host_info->bdf = rid;
2916 host_info->os_type = ENA_ADMIN_OS_FREEBSD;
2917 host_info->kernel_ver = osreldate;
2918
2919 sprintf(host_info->kernel_ver_str, "%d", osreldate);
2920 host_info->os_dist = 0;
2921 strncpy(host_info->os_dist_str, osrelease,
2922 sizeof(host_info->os_dist_str) - 1);
2923
2924 host_info->driver_version = (ENA_DRV_MODULE_VER_MAJOR) |
2925 (ENA_DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
2926 (ENA_DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
2927 host_info->num_cpus = mp_ncpus;
2928 host_info->driver_supported_features =
2929 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
2930 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
2931
2932 rc = ena_com_set_host_attributes(ena_dev);
2933 if (unlikely(rc != 0)) {
2934 if (rc == EOPNOTSUPP)
2935 ena_log(dev, WARN, "Cannot set host attributes\n");
2936 else
2937 ena_log(dev, ERR, "Cannot set host attributes\n");
2938
2939 goto err;
2940 }
2941
2942 return;
2943
2944 err:
2945 ena_com_delete_host_info(ena_dev);
2946 }
2947
2948 static int
ena_device_init(struct ena_adapter * adapter,device_t pdev,struct ena_com_dev_get_features_ctx * get_feat_ctx,int * wd_active)2949 ena_device_init(struct ena_adapter *adapter, device_t pdev,
2950 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active)
2951 {
2952 struct ena_llq_configurations llq_config;
2953 struct ena_com_dev *ena_dev = adapter->ena_dev;
2954 bool readless_supported;
2955 uint32_t aenq_groups;
2956 int dma_width;
2957 int rc;
2958
2959 rc = ena_com_mmio_reg_read_request_init(ena_dev);
2960 if (unlikely(rc != 0)) {
2961 ena_log(pdev, ERR, "failed to init mmio read less\n");
2962 return (rc);
2963 }
2964
2965 /*
2966 * The PCIe configuration space revision id indicate if mmio reg
2967 * read is disabled
2968 */
2969 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ);
2970 ena_com_set_mmio_read_mode(ena_dev, readless_supported);
2971
2972 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
2973 if (unlikely(rc != 0)) {
2974 ena_log(pdev, ERR, "Can not reset device\n");
2975 goto err_mmio_read_less;
2976 }
2977
2978 rc = ena_com_validate_version(ena_dev);
2979 if (unlikely(rc != 0)) {
2980 ena_log(pdev, ERR, "device version is too low\n");
2981 goto err_mmio_read_less;
2982 }
2983
2984 dma_width = ena_com_get_dma_width(ena_dev);
2985 if (unlikely(dma_width < 0)) {
2986 ena_log(pdev, ERR, "Invalid dma width value %d", dma_width);
2987 rc = dma_width;
2988 goto err_mmio_read_less;
2989 }
2990 adapter->dma_width = dma_width;
2991
2992 /* ENA admin level init */
2993 rc = ena_com_admin_init(ena_dev, &aenq_handlers);
2994 if (unlikely(rc != 0)) {
2995 ena_log(pdev, ERR,
2996 "Can not initialize ena admin queue with device\n");
2997 goto err_mmio_read_less;
2998 }
2999
3000 /*
3001 * To enable the msix interrupts the driver needs to know the number
3002 * of queues. So the driver uses polling mode to retrieve this
3003 * information
3004 */
3005 ena_com_set_admin_polling_mode(ena_dev, true);
3006
3007 ena_config_host_info(ena_dev, pdev);
3008
3009 /* Get Device Attributes */
3010 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3011 if (unlikely(rc != 0)) {
3012 ena_log(pdev, ERR,
3013 "Cannot get attribute for ena device rc: %d\n", rc);
3014 goto err_admin_init;
3015 }
3016
3017 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3018 BIT(ENA_ADMIN_FATAL_ERROR) |
3019 BIT(ENA_ADMIN_WARNING) |
3020 BIT(ENA_ADMIN_NOTIFICATION) |
3021 BIT(ENA_ADMIN_KEEP_ALIVE) |
3022 BIT(ENA_ADMIN_CONF_NOTIFICATIONS) |
3023 BIT(ENA_ADMIN_DEVICE_REQUEST_RESET);
3024
3025 aenq_groups &= get_feat_ctx->aenq.supported_groups;
3026 rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3027 if (unlikely(rc != 0)) {
3028 ena_log(pdev, ERR, "Cannot configure aenq groups rc: %d\n", rc);
3029 goto err_admin_init;
3030 }
3031
3032 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3033
3034 ena_set_llq_configurations(&llq_config, &get_feat_ctx->llq, adapter);
3035
3036 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3037 &llq_config);
3038 if (unlikely(rc != 0)) {
3039 ena_log(pdev, ERR, "Failed to set placement policy\n");
3040 goto err_admin_init;
3041 }
3042
3043 return (0);
3044
3045 err_admin_init:
3046 ena_com_delete_host_info(ena_dev);
3047 ena_com_admin_destroy(ena_dev);
3048 err_mmio_read_less:
3049 ena_com_mmio_reg_read_request_destroy(ena_dev);
3050
3051 return (rc);
3052 }
3053
3054 static int
ena_enable_msix_and_set_admin_interrupts(struct ena_adapter * adapter)3055 ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3056 {
3057 struct ena_com_dev *ena_dev = adapter->ena_dev;
3058 int rc;
3059
3060 rc = ena_enable_msix(adapter);
3061 if (unlikely(rc != 0)) {
3062 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n");
3063 return (rc);
3064 }
3065
3066 ena_setup_mgmnt_intr(adapter);
3067
3068 rc = ena_request_mgmnt_irq(adapter);
3069 if (unlikely(rc != 0)) {
3070 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n");
3071 goto err_disable_msix;
3072 }
3073
3074 ena_com_set_admin_polling_mode(ena_dev, false);
3075
3076 ena_com_admin_aenq_enable(ena_dev);
3077
3078 return (0);
3079
3080 err_disable_msix:
3081 ena_disable_msix(adapter);
3082
3083 return (rc);
3084 }
3085
3086 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
3087 static void
ena_keep_alive_wd(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)3088 ena_keep_alive_wd(void *adapter_data, struct ena_admin_aenq_entry *aenq_e)
3089 {
3090 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3091 struct ena_admin_aenq_keep_alive_desc *desc;
3092 sbintime_t stime;
3093 uint64_t rx_drops;
3094 uint64_t tx_drops;
3095
3096 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3097
3098 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
3099 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low;
3100 counter_u64_zero(adapter->hw_stats.rx_drops);
3101 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops);
3102 counter_u64_zero(adapter->hw_stats.tx_drops);
3103 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops);
3104
3105 stime = getsbinuptime();
3106 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime);
3107 }
3108
3109 /* Check for keep alive expiration */
3110 static void
check_for_missing_keep_alive(struct ena_adapter * adapter)3111 check_for_missing_keep_alive(struct ena_adapter *adapter)
3112 {
3113 sbintime_t timestamp, time;
3114 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
3115
3116 if (adapter->wd_active == 0)
3117 return;
3118
3119 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3120 return;
3121
3122 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp);
3123 time = getsbinuptime() - timestamp;
3124 if (unlikely(time > adapter->keep_alive_timeout)) {
3125 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n");
3126 if (ena_com_aenq_has_keep_alive(adapter->ena_dev))
3127 reset_reason = ENA_REGS_RESET_MISSING_ADMIN_INTERRUPT;
3128
3129 ena_trigger_reset(adapter, reset_reason);
3130 }
3131 }
3132
3133 /* Check if admin queue is enabled */
3134 static void
check_for_admin_com_state(struct ena_adapter * adapter)3135 check_for_admin_com_state(struct ena_adapter *adapter)
3136 {
3137 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_ADMIN_TO;
3138 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == false)) {
3139 ena_log(adapter->pdev, ERR,
3140 "ENA admin queue is not in running state!\n");
3141 counter_u64_add(adapter->dev_stats.admin_q_pause, 1);
3142 if (ena_com_get_missing_admin_interrupt(adapter->ena_dev))
3143 reset_reason = ENA_REGS_RESET_MISSING_ADMIN_INTERRUPT;
3144
3145 ena_trigger_reset(adapter, reset_reason);
3146 }
3147 }
3148
3149 static int
check_for_rx_interrupt_queue(struct ena_adapter * adapter,struct ena_ring * rx_ring)3150 check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3151 struct ena_ring *rx_ring)
3152 {
3153 if (likely(atomic_load_8(&rx_ring->first_interrupt)))
3154 return (0);
3155
3156 if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3157 return (0);
3158
3159 rx_ring->no_interrupt_event_cnt++;
3160
3161 if (rx_ring->no_interrupt_event_cnt ==
3162 ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3163 ena_log(adapter->pdev, ERR,
3164 "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3165 rx_ring->qid);
3166 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3167 return (EIO);
3168 }
3169
3170 return (0);
3171 }
3172
3173 static enum ena_regs_reset_reason_types
check_cdesc_in_tx_cq(struct ena_adapter * adapter,struct ena_ring * tx_ring)3174 check_cdesc_in_tx_cq(struct ena_adapter *adapter,
3175 struct ena_ring *tx_ring)
3176 {
3177 device_t pdev = adapter->pdev;
3178 int rc;
3179 u16 req_id;
3180
3181 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id);
3182 /* TX CQ is empty */
3183 if (rc == ENA_COM_TRY_AGAIN) {
3184 ena_log(pdev, ERR,
3185 "No completion descriptors found in CQ %d\n",
3186 tx_ring->qid);
3187 return ENA_REGS_RESET_MISS_TX_CMPL;
3188 }
3189
3190 /* TX CQ has cdescs */
3191 ena_log(pdev, ERR,
3192 "Completion descriptors found in CQ %d",
3193 tx_ring->qid);
3194
3195 return ENA_REGS_RESET_MISS_INTERRUPT;
3196 }
3197
3198 static int
check_missing_comp_in_tx_queue(struct ena_adapter * adapter,struct ena_ring * tx_ring)3199 check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3200 struct ena_ring *tx_ring)
3201 {
3202 uint32_t missed_tx = 0, new_missed_tx = 0;
3203 device_t pdev = adapter->pdev;
3204 struct bintime curtime, time;
3205 struct ena_tx_buffer *tx_buf;
3206 int time_since_last_cleanup;
3207 int missing_tx_comp_to;
3208 sbintime_t time_offset;
3209 int i, rc = 0;
3210 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;
3211 bool cleanup_scheduled, cleanup_running;
3212
3213 getbinuptime(&curtime);
3214
3215 for (i = 0; i < tx_ring->ring_size; i++) {
3216 tx_buf = &tx_ring->tx_buffer_info[i];
3217
3218 if (bintime_isset(&tx_buf->timestamp) == 0)
3219 continue;
3220
3221 time = curtime;
3222 bintime_sub(&time, &tx_buf->timestamp);
3223 time_offset = bttosbt(time);
3224
3225 if (unlikely(!atomic_load_8(&tx_ring->first_interrupt) &&
3226 time_offset > 2 * adapter->missing_tx_timeout)) {
3227 /*
3228 * If after graceful period interrupt is still not
3229 * received, we schedule a reset.
3230 */
3231 ena_log(pdev, ERR,
3232 "Potential MSIX issue on Tx side Queue = %d. "
3233 "Reset the device\n",
3234 tx_ring->qid);
3235 ena_trigger_reset(adapter,
3236 ENA_REGS_RESET_MISS_INTERRUPT);
3237 return (EIO);
3238 }
3239
3240 /* Check again if packet is still waiting */
3241 if (unlikely(time_offset > adapter->missing_tx_timeout)) {
3242
3243 if (tx_buf->print_once) {
3244 time_since_last_cleanup = TICKS_2_MSEC(ticks -
3245 tx_ring->tx_last_cleanup_ticks);
3246 missing_tx_comp_to = sbttoms(
3247 adapter->missing_tx_timeout);
3248 ena_log(pdev, WARN,
3249 "Found a Tx that wasn't completed on time, qid %d, index %d. "
3250 "%d msecs have passed since last cleanup. Missing Tx timeout value %d msecs.\n",
3251 tx_ring->qid, i, time_since_last_cleanup,
3252 missing_tx_comp_to);
3253 /* Add new TX completions which are missed */
3254 new_missed_tx++;
3255 }
3256
3257 tx_buf->print_once = false;
3258 missed_tx++;
3259 }
3260 }
3261 /* Checking if this TX ring missing TX completions have passed the threshold */
3262 if (unlikely(missed_tx > adapter->missing_tx_threshold)) {
3263 ena_log(pdev, ERR,
3264 "The number of lost tx completion is above the threshold "
3265 "(%d > %d). Reset the device\n",
3266 missed_tx, adapter->missing_tx_threshold);
3267 /* Set the reset flag to prevent ena_cleanup() from running */
3268 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
3269 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and
3270 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to
3271 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq()
3272 */
3273 mb();
3274 cleanup_scheduled = !!(atomic_load_16(&tx_ring->que->cleanup_task.ta_pending));
3275 cleanup_running = !!(atomic_load_8((&tx_ring->cleanup_running)));
3276 if (!(cleanup_scheduled || cleanup_running))
3277 reset_reason = check_cdesc_in_tx_cq(adapter, tx_ring);
3278
3279 adapter->reset_reason = reset_reason;
3280 rc = EIO;
3281 }
3282 /* Add the newly discovered missing TX completions */
3283 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, new_missed_tx);
3284
3285 return (rc);
3286 }
3287
3288 /*
3289 * Check for TX which were not completed on time.
3290 * Timeout is defined by "missing_tx_timeout".
3291 * Reset will be performed if number of incompleted
3292 * transactions exceeds "missing_tx_threshold".
3293 */
3294 static void
check_for_missing_completions(struct ena_adapter * adapter)3295 check_for_missing_completions(struct ena_adapter *adapter)
3296 {
3297 struct ena_ring *tx_ring;
3298 struct ena_ring *rx_ring;
3299 int i, budget, rc;
3300
3301 /* Make sure the driver doesn't turn the device in other process */
3302 rmb();
3303
3304 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3305 return;
3306
3307 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
3308 return;
3309
3310 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3311 return;
3312
3313 budget = adapter->missing_tx_max_queues;
3314
3315 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) {
3316 tx_ring = &adapter->tx_ring[i];
3317 rx_ring = &adapter->rx_ring[i];
3318
3319 rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3320 if (unlikely(rc != 0))
3321 return;
3322
3323 rc = check_for_rx_interrupt_queue(adapter, rx_ring);
3324 if (unlikely(rc != 0))
3325 return;
3326
3327 budget--;
3328 if (budget == 0) {
3329 i++;
3330 break;
3331 }
3332 }
3333
3334 adapter->next_monitored_tx_qid = i % adapter->num_io_queues;
3335 }
3336
3337 /* trigger rx cleanup after 2 consecutive detections */
3338 #define EMPTY_RX_REFILL 2
3339 /* For the rare case where the device runs out of Rx descriptors and the
3340 * msix handler failed to refill new Rx descriptors (due to a lack of memory
3341 * for example).
3342 * This case will lead to a deadlock:
3343 * The device won't send interrupts since all the new Rx packets will be dropped
3344 * The msix handler won't allocate new Rx descriptors so the device won't be
3345 * able to send new packets.
3346 *
3347 * When such a situation is detected - execute rx cleanup task in another thread
3348 */
3349 static void
check_for_empty_rx_ring(struct ena_adapter * adapter)3350 check_for_empty_rx_ring(struct ena_adapter *adapter)
3351 {
3352 struct ena_ring *rx_ring;
3353 int i, refill_required;
3354
3355 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3356 return;
3357
3358 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
3359 return;
3360
3361 for (i = 0; i < adapter->num_io_queues; i++) {
3362 rx_ring = &adapter->rx_ring[i];
3363
3364 refill_required = ena_com_free_q_entries(
3365 rx_ring->ena_com_io_sq);
3366 if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3367 rx_ring->empty_rx_queue++;
3368
3369 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3370 counter_u64_add(rx_ring->rx_stats.empty_rx_ring,
3371 1);
3372
3373 ena_log(adapter->pdev, WARN,
3374 "Rx ring %d is stalled. Triggering the refill function\n",
3375 i);
3376
3377 taskqueue_enqueue(rx_ring->que->cleanup_tq,
3378 &rx_ring->que->cleanup_task);
3379 rx_ring->empty_rx_queue = 0;
3380 }
3381 } else {
3382 rx_ring->empty_rx_queue = 0;
3383 }
3384 }
3385 }
3386
3387 static void
ena_update_hints(struct ena_adapter * adapter,struct ena_admin_ena_hw_hints * hints)3388 ena_update_hints(struct ena_adapter *adapter,
3389 struct ena_admin_ena_hw_hints *hints)
3390 {
3391 struct ena_com_dev *ena_dev = adapter->ena_dev;
3392
3393 if (hints->admin_completion_tx_timeout)
3394 ena_dev->admin_queue.completion_timeout =
3395 hints->admin_completion_tx_timeout * 1000;
3396
3397 if (hints->mmio_read_timeout)
3398 /* convert to usec */
3399 ena_dev->mmio_read.reg_read_to = hints->mmio_read_timeout * 1000;
3400
3401 if (hints->missed_tx_completion_count_threshold_to_reset)
3402 adapter->missing_tx_threshold =
3403 hints->missed_tx_completion_count_threshold_to_reset;
3404
3405 if (hints->missing_tx_completion_timeout) {
3406 if (hints->missing_tx_completion_timeout ==
3407 ENA_HW_HINTS_NO_TIMEOUT)
3408 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3409 else
3410 adapter->missing_tx_timeout = SBT_1MS *
3411 hints->missing_tx_completion_timeout;
3412 }
3413
3414 if (hints->driver_watchdog_timeout) {
3415 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3416 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3417 else
3418 adapter->keep_alive_timeout = SBT_1MS *
3419 hints->driver_watchdog_timeout;
3420 }
3421 }
3422
3423 /**
3424 * ena_copy_eni_metrics - Get and copy ENI metrics from the HW.
3425 * @adapter: ENA device adapter
3426 *
3427 * Returns 0 on success, EOPNOTSUPP if current HW doesn't support those metrics
3428 * and other error codes on failure.
3429 *
3430 * This function can possibly cause a race with other calls to the admin queue.
3431 * Because of that, the caller should either lock this function or make sure
3432 * that there is no race in the current context.
3433 */
3434 static int
ena_copy_eni_metrics(struct ena_adapter * adapter)3435 ena_copy_eni_metrics(struct ena_adapter *adapter)
3436 {
3437 static bool print_once = true;
3438 int rc;
3439
3440 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics);
3441
3442 if (rc != 0) {
3443 if (rc == ENA_COM_UNSUPPORTED) {
3444 if (print_once) {
3445 ena_log(adapter->pdev, WARN,
3446 "Retrieving ENI metrics is not supported.\n");
3447 print_once = false;
3448 } else {
3449 ena_log(adapter->pdev, DBG,
3450 "Retrieving ENI metrics is not supported.\n");
3451 }
3452 } else {
3453 ena_log(adapter->pdev, ERR,
3454 "Failed to get ENI metrics: %d\n", rc);
3455 }
3456 }
3457
3458 return (rc);
3459 }
3460
3461 static int
ena_copy_srd_metrics(struct ena_adapter * adapter)3462 ena_copy_srd_metrics(struct ena_adapter *adapter)
3463 {
3464 return ena_com_get_ena_srd_info(adapter->ena_dev, &adapter->ena_srd_info);
3465 }
3466
3467 static int
ena_copy_customer_metrics(struct ena_adapter * adapter)3468 ena_copy_customer_metrics(struct ena_adapter *adapter)
3469 {
3470 struct ena_com_dev *dev;
3471 u32 supported_metrics_count;
3472 int rc, len;
3473
3474 dev = adapter->ena_dev;
3475
3476 supported_metrics_count = ena_com_get_customer_metric_count(dev);
3477 len = supported_metrics_count * sizeof(u64);
3478
3479 /* Fill the data buffer */
3480 rc = ena_com_get_customer_metrics(adapter->ena_dev,
3481 (char *)(adapter->customer_metrics_array), len);
3482
3483 return (rc);
3484 }
3485
3486 static void
ena_timer_service(void * data)3487 ena_timer_service(void *data)
3488 {
3489 struct ena_adapter *adapter = (struct ena_adapter *)data;
3490 struct ena_admin_host_info *host_info =
3491 adapter->ena_dev->host_attr.host_info;
3492
3493 check_for_missing_keep_alive(adapter);
3494
3495 check_for_admin_com_state(adapter);
3496
3497 check_for_missing_completions(adapter);
3498
3499 check_for_empty_rx_ring(adapter);
3500
3501 /*
3502 * User controller update of the ENA metrics.
3503 * If the delay was set to 0, then the stats shouldn't be updated at
3504 * all.
3505 * Otherwise, wait 'metrics_sample_interval' seconds, before
3506 * updating stats.
3507 * As timer service is executed every second, it's enough to increment
3508 * appropriate counter each time the timer service is executed.
3509 */
3510 if ((adapter->metrics_sample_interval != 0) &&
3511 (++adapter->metrics_sample_interval_cnt >=
3512 adapter->metrics_sample_interval)) {
3513 taskqueue_enqueue(adapter->metrics_tq, &adapter->metrics_task);
3514 adapter->metrics_sample_interval_cnt = 0;
3515 }
3516
3517
3518 if (host_info != NULL)
3519 ena_update_host_info(host_info, adapter->ifp);
3520
3521 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
3522 /*
3523 * Timeout when validating version indicates that the device
3524 * became unresponsive. If that happens skip the reset and
3525 * reschedule timer service, so the reset can be retried later.
3526 */
3527 if (ena_com_validate_version(adapter->ena_dev) ==
3528 ENA_COM_TIMER_EXPIRED) {
3529 ena_log(adapter->pdev, WARN,
3530 "FW unresponsive, skipping reset\n");
3531 ENA_TIMER_RESET(adapter);
3532 return;
3533 }
3534 ena_log(adapter->pdev, WARN, "Trigger reset is on\n");
3535 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3536 return;
3537 }
3538
3539 /*
3540 * Schedule another timeout one second from now.
3541 */
3542 ENA_TIMER_RESET(adapter);
3543 }
3544
3545 void
ena_destroy_device(struct ena_adapter * adapter,bool graceful)3546 ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3547 {
3548 if_t ifp = adapter->ifp;
3549 struct ena_com_dev *ena_dev = adapter->ena_dev;
3550 bool dev_up;
3551
3552 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))
3553 return;
3554
3555 if (!graceful)
3556 if_link_state_change(ifp, LINK_STATE_DOWN);
3557
3558 ENA_TIMER_DRAIN(adapter);
3559
3560 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
3561 if (dev_up)
3562 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3563
3564 if (!graceful)
3565 ena_com_set_admin_running_state(ena_dev, false);
3566
3567 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3568 ena_down(adapter);
3569
3570 /*
3571 * Stop the device from sending AENQ events (if the device was up, and
3572 * the trigger reset was on, ena_down already performs device reset)
3573 */
3574 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up))
3575 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3576
3577 ena_free_mgmnt_irq(adapter);
3578
3579 ena_disable_msix(adapter);
3580
3581 /*
3582 * IO rings resources should be freed because `ena_restore_device()`
3583 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX
3584 * vectors. The amount of MSIX vectors after destroy-restore may be
3585 * different than before. Therefore, IO rings resources should be
3586 * established from scratch each time.
3587 */
3588 ena_free_all_io_rings_resources(adapter);
3589
3590 ena_com_abort_admin_commands(ena_dev);
3591
3592 ena_com_wait_for_abort_completion(ena_dev);
3593
3594 ena_com_admin_destroy(ena_dev);
3595
3596 ena_com_mmio_reg_read_request_destroy(ena_dev);
3597
3598 adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3599
3600 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
3601 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3602 }
3603
3604 static int
ena_device_validate_params(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * get_feat_ctx)3605 ena_device_validate_params(struct ena_adapter *adapter,
3606 struct ena_com_dev_get_features_ctx *get_feat_ctx)
3607 {
3608 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr,
3609 ETHER_ADDR_LEN) != 0) {
3610 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n");
3611 return (EINVAL);
3612 }
3613
3614 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) {
3615 ena_log(adapter->pdev, ERR,
3616 "Error, device max mtu is smaller than ifp MTU\n");
3617 return (EINVAL);
3618 }
3619
3620 return 0;
3621 }
3622
3623 int
ena_restore_device(struct ena_adapter * adapter)3624 ena_restore_device(struct ena_adapter *adapter)
3625 {
3626 struct ena_com_dev_get_features_ctx get_feat_ctx;
3627 struct ena_com_dev *ena_dev = adapter->ena_dev;
3628 if_t ifp = adapter->ifp;
3629 device_t dev = adapter->pdev;
3630 int wd_active;
3631 int rc;
3632
3633 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3634
3635 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active);
3636 if (rc != 0) {
3637 ena_log(dev, ERR, "Cannot initialize device\n");
3638 goto err;
3639 }
3640 /*
3641 * Only enable WD if it was enabled before reset, so it won't override
3642 * value set by the user by the sysctl.
3643 */
3644 if (adapter->wd_active != 0)
3645 adapter->wd_active = wd_active;
3646
3647 rc = ena_device_validate_params(adapter, &get_feat_ctx);
3648 if (rc != 0) {
3649 ena_log(dev, ERR, "Validation of device parameters failed\n");
3650 goto err_device_destroy;
3651 }
3652
3653 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3654 /* Make sure we don't have a race with AENQ Links state handler */
3655 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
3656 if_link_state_change(ifp, LINK_STATE_UP);
3657
3658 rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3659 if (rc != 0) {
3660 ena_log(dev, ERR, "Enable MSI-X failed\n");
3661 goto err_device_destroy;
3662 }
3663
3664 /*
3665 * Effective value of used MSIX vectors should be the same as before
3666 * `ena_destroy_device()`, if possible, or closest to it if less vectors
3667 * are available.
3668 */
3669 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues)
3670 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC;
3671
3672 /* Re-initialize rings basic information */
3673 ena_init_io_rings(adapter);
3674
3675 /* If the interface was up before the reset bring it up */
3676 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) {
3677 rc = ena_up(adapter);
3678 if (rc != 0) {
3679 ena_log(dev, ERR, "Failed to create I/O queues\n");
3680 goto err_disable_msix;
3681 }
3682 }
3683
3684 /* Indicate that device is running again and ready to work */
3685 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3686
3687 /*
3688 * As the AENQ handlers weren't executed during reset because
3689 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the
3690 * timestamp must be updated again That will prevent next reset
3691 * caused by missing keep alive.
3692 */
3693 adapter->keep_alive_timestamp = getsbinuptime();
3694 ENA_TIMER_RESET(adapter);
3695
3696 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3697
3698 return (rc);
3699
3700 err_disable_msix:
3701 ena_free_mgmnt_irq(adapter);
3702 ena_disable_msix(adapter);
3703 err_device_destroy:
3704 ena_com_abort_admin_commands(ena_dev);
3705 ena_com_wait_for_abort_completion(ena_dev);
3706 ena_com_admin_destroy(ena_dev);
3707 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3708 ena_com_mmio_reg_read_request_destroy(ena_dev);
3709 err:
3710 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3711 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3712 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n");
3713
3714 return (rc);
3715 }
3716
3717 static void
ena_metrics_task(void * arg,int pending)3718 ena_metrics_task(void *arg, int pending)
3719 {
3720 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3721
3722 ENA_LOCK_LOCK();
3723
3724 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_CUSTOMER_METRICS))
3725 (void)ena_copy_customer_metrics(adapter);
3726 else if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS))
3727 (void)ena_copy_eni_metrics(adapter);
3728
3729 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENA_SRD_INFO))
3730 (void)ena_copy_srd_metrics(adapter);
3731
3732 ENA_LOCK_UNLOCK();
3733 }
3734
3735 static void
ena_reset_task(void * arg,int pending)3736 ena_reset_task(void *arg, int pending)
3737 {
3738 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3739
3740 ENA_LOCK_LOCK();
3741 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
3742 ena_increment_reset_counter(adapter);
3743 ena_destroy_device(adapter, false);
3744 ena_restore_device(adapter);
3745
3746 ena_log(adapter->pdev, INFO,
3747 "Device reset completed successfully, Driver info: %s\n",
3748 ena_version);
3749 }
3750 ENA_LOCK_UNLOCK();
3751 }
3752
3753 static void
ena_free_stats(struct ena_adapter * adapter)3754 ena_free_stats(struct ena_adapter *adapter)
3755 {
3756 ena_free_counters((counter_u64_t *)&adapter->hw_stats,
3757 sizeof(struct ena_hw_stats));
3758 ena_free_counters((counter_u64_t *)&adapter->dev_stats,
3759 sizeof(struct ena_stats_dev));
3760
3761 }
3762 /**
3763 * ena_attach - Device Initialization Routine
3764 * @pdev: device information struct
3765 *
3766 * Returns 0 on success, otherwise on failure.
3767 *
3768 * ena_attach initializes an adapter identified by a device structure.
3769 * The OS initialization, configuring of the adapter private structure,
3770 * and a hardware reset occur.
3771 **/
3772 static int
ena_attach(device_t pdev)3773 ena_attach(device_t pdev)
3774 {
3775 struct ena_com_dev_get_features_ctx get_feat_ctx;
3776 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
3777 static int version_printed;
3778 struct ena_adapter *adapter;
3779 struct ena_com_dev *ena_dev = NULL;
3780 uint32_t max_num_io_queues;
3781 int msix_rid;
3782 int rid, rc;
3783
3784 adapter = device_get_softc(pdev);
3785 adapter->pdev = pdev;
3786 adapter->first_bind = -1;
3787
3788 /*
3789 * Set up the timer service - driver is responsible for avoiding
3790 * concurrency, as the callout won't be using any locking inside.
3791 */
3792 ENA_TIMER_INIT(adapter);
3793 adapter->keep_alive_timeout = ENA_DEFAULT_KEEP_ALIVE_TO;
3794 adapter->missing_tx_timeout = ENA_DEFAULT_TX_CMP_TO;
3795 adapter->missing_tx_max_queues = ENA_DEFAULT_TX_MONITORED_QUEUES;
3796 adapter->missing_tx_threshold = ENA_DEFAULT_TX_CMP_THRESHOLD;
3797
3798 adapter->irq_cpu_base = ENA_BASE_CPU_UNSPECIFIED;
3799 adapter->irq_cpu_stride = 0;
3800
3801 #ifdef RSS
3802 adapter->rss_enabled = 1;
3803 #endif
3804
3805 if (version_printed++ == 0)
3806 ena_log(pdev, INFO, "%s\n", ena_version);
3807
3808 /* Allocate memory for ena_dev structure */
3809 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF,
3810 M_WAITOK | M_ZERO);
3811
3812 adapter->ena_dev = ena_dev;
3813 ena_dev->dmadev = pdev;
3814
3815 rid = PCIR_BAR(ENA_REG_BAR);
3816 adapter->memory = NULL;
3817 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid,
3818 RF_ACTIVE);
3819 if (unlikely(adapter->registers == NULL)) {
3820 ena_log(pdev, ERR,
3821 "unable to allocate bus resource: registers!\n");
3822 rc = ENOMEM;
3823 goto err_dev_free;
3824 }
3825
3826 /* MSIx vector table may reside on BAR0 with registers or on BAR1. */
3827 msix_rid = pci_msix_table_bar(pdev);
3828 if (msix_rid != rid) {
3829 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
3830 &msix_rid, RF_ACTIVE);
3831 if (unlikely(adapter->msix == NULL)) {
3832 ena_log(pdev, ERR,
3833 "unable to allocate bus resource: msix!\n");
3834 rc = ENOMEM;
3835 goto err_pci_free;
3836 }
3837 adapter->msix_rid = msix_rid;
3838 }
3839
3840 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3841 M_WAITOK | M_ZERO);
3842
3843 /* Store register resources */
3844 ((struct ena_bus *)(ena_dev->bus))->reg_bar_t = rman_get_bustag(
3845 adapter->registers);
3846 ((struct ena_bus *)(ena_dev->bus))->reg_bar_h = rman_get_bushandle(
3847 adapter->registers);
3848
3849 if (unlikely(((struct ena_bus *)(ena_dev->bus))->reg_bar_h == 0)) {
3850 ena_log(pdev, ERR, "failed to pmap registers bar\n");
3851 rc = ENXIO;
3852 goto err_bus_free;
3853 }
3854
3855 rc = ena_map_llq_mem_bar(pdev, ena_dev);
3856 if (unlikely(rc != 0)) {
3857 ena_log(pdev, ERR, "Failed to map ENA mem bar");
3858 goto err_bus_free;
3859 }
3860
3861 ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
3862
3863 /* Initially clear all the flags */
3864 ENA_FLAG_ZERO(adapter);
3865
3866 /* Device initialization */
3867 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3868 if (unlikely(rc != 0)) {
3869 ena_log(pdev, ERR, "ENA device init failed! (err: %d)\n", rc);
3870 rc = ENXIO;
3871 goto err_bus_free;
3872 }
3873
3874 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3875 adapter->disable_meta_caching = !!(
3876 get_feat_ctx.llq.accel_mode.u.get.supported_flags &
3877 BIT(ENA_ADMIN_DISABLE_META_CACHING));
3878
3879 adapter->keep_alive_timestamp = getsbinuptime();
3880
3881 adapter->tx_offload_cap = get_feat_ctx.offload.tx;
3882
3883 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr,
3884 ETHER_ADDR_LEN);
3885
3886 calc_queue_ctx.pdev = pdev;
3887 calc_queue_ctx.ena_dev = ena_dev;
3888 calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
3889
3890 /* Calculate initial and maximum IO queue number and size */
3891 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev,
3892 &get_feat_ctx);
3893 rc = ena_calc_io_queue_size(&calc_queue_ctx, adapter);
3894 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) {
3895 rc = EFAULT;
3896 goto err_com_free;
3897 }
3898
3899 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size;
3900 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size;
3901 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
3902 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
3903 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
3904 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3905
3906 adapter->max_num_io_queues = max_num_io_queues;
3907
3908 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE;
3909
3910 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
3911
3912 adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3913
3914 /* set up dma tags for rx and tx buffers */
3915 rc = ena_setup_tx_dma_tag(adapter);
3916 if (unlikely(rc != 0)) {
3917 ena_log(pdev, ERR, "Failed to create TX DMA tag\n");
3918 goto err_com_free;
3919 }
3920
3921 rc = ena_setup_rx_dma_tag(adapter);
3922 if (unlikely(rc != 0)) {
3923 ena_log(pdev, ERR, "Failed to create RX DMA tag\n");
3924 goto err_tx_tag_free;
3925 }
3926
3927 /*
3928 * The amount of requested MSIX vectors is equal to
3929 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant
3930 * number of admin queue interrupts. The former is initially determined
3931 * by HW capabilities (see `ena_calc_max_io_queue_num())` but may not be
3932 * achieved if there are not enough system resources. By default, the
3933 * number of effectively used IO queues is the same but later on it can
3934 * be limited by the user using sysctl interface.
3935 */
3936 rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3937 if (unlikely(rc != 0)) {
3938 ena_log(pdev, ERR,
3939 "Failed to enable and set the admin interrupts\n");
3940 goto err_io_free;
3941 }
3942 /* By default all of allocated MSIX vectors are actively used */
3943 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC;
3944
3945 /* initialize rings basic information */
3946 ena_init_io_rings(adapter);
3947
3948 rc = ena_com_allocate_customer_metrics_buffer(ena_dev);
3949 if (rc) {
3950 ena_log(pdev, ERR, "Failed to allocate customer metrics buffer.\n");
3951 goto err_msix_free;
3952 }
3953
3954 rc = ena_sysctl_allocate_customer_metrics_buffer(adapter);
3955 if (unlikely(rc)){
3956 ena_log(pdev, ERR, "Failed to allocate sysctl customer metrics buffer.\n");
3957 goto err_metrics_buffer_destroy;
3958 }
3959
3960 /* Initialize statistics */
3961 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3962 sizeof(struct ena_stats_dev));
3963 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3964 sizeof(struct ena_hw_stats));
3965 ena_sysctl_add_nodes(adapter);
3966
3967 /* setup network interface */
3968 ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3969
3970 /* Initialize reset task queue */
3971 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3972 adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3973 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3974 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, "%s rstq",
3975 device_get_nameunit(adapter->pdev));
3976
3977 /* Initialize metrics task queue */
3978 TASK_INIT(&adapter->metrics_task, 0, ena_metrics_task, adapter);
3979 adapter->metrics_tq = taskqueue_create("ena_metrics_enqueue",
3980 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->metrics_tq);
3981 taskqueue_start_threads(&adapter->metrics_tq, 1, PI_NET, "%s metricsq",
3982 device_get_nameunit(adapter->pdev));
3983
3984 #ifdef DEV_NETMAP
3985 rc = ena_netmap_attach(adapter);
3986 if (rc != 0) {
3987 ena_log(pdev, ERR, "netmap attach failed: %d\n", rc);
3988 goto err_detach;
3989 }
3990 #endif /* DEV_NETMAP */
3991
3992 /* Tell the stack that the interface is not active */
3993 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3994 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3995
3996 /* Run the timer service */
3997 ENA_TIMER_RESET(adapter);
3998
3999 return (0);
4000
4001 #ifdef DEV_NETMAP
4002 err_detach:
4003 ether_ifdetach(adapter->ifp);
4004 ifmedia_removeall(&adapter->media);
4005 free(adapter->customer_metrics_array, M_DEVBUF);
4006 #endif /* DEV_NETMAP */
4007 err_metrics_buffer_destroy:
4008 ena_com_delete_customer_metrics_buffer(ena_dev);
4009 err_msix_free:
4010 ena_free_stats(adapter);
4011 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR);
4012 ena_free_mgmnt_irq(adapter);
4013 ena_disable_msix(adapter);
4014 err_io_free:
4015 ena_free_all_io_rings_resources(adapter);
4016 ena_free_rx_dma_tag(adapter);
4017 err_tx_tag_free:
4018 ena_free_tx_dma_tag(adapter);
4019 err_com_free:
4020 ena_com_admin_destroy(ena_dev);
4021 ena_com_delete_host_info(ena_dev);
4022 ena_com_mmio_reg_read_request_destroy(ena_dev);
4023 err_bus_free:
4024 free(ena_dev->bus, M_DEVBUF);
4025 err_pci_free:
4026 ena_free_pci_resources(adapter);
4027 err_dev_free:
4028 free(ena_dev, M_DEVBUF);
4029
4030 return (rc);
4031 }
4032
4033 /**
4034 * ena_detach - Device Removal Routine
4035 * @pdev: device information struct
4036 *
4037 * ena_detach is called by the device subsystem to alert the driver
4038 * that it should release a PCI device.
4039 **/
4040 static int
ena_detach(device_t pdev)4041 ena_detach(device_t pdev)
4042 {
4043 struct ena_adapter *adapter = device_get_softc(pdev);
4044 struct ena_com_dev *ena_dev = adapter->ena_dev;
4045 int rc;
4046
4047 /* Make sure VLANS are not using driver */
4048 if (adapter->ifp->if_vlantrunk != NULL) {
4049 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n");
4050 return (EBUSY);
4051 }
4052
4053 ether_ifdetach(adapter->ifp);
4054
4055 ifmedia_removeall(&adapter->media);
4056
4057 /* Stop timer service */
4058 ENA_LOCK_LOCK();
4059 ENA_TIMER_DRAIN(adapter);
4060 ENA_LOCK_UNLOCK();
4061
4062 /* Release metrics task */
4063 while (taskqueue_cancel(adapter->metrics_tq, &adapter->metrics_task, NULL))
4064 taskqueue_drain(adapter->metrics_tq, &adapter->metrics_task);
4065 taskqueue_free(adapter->metrics_tq);
4066
4067 /* Release reset task */
4068 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
4069 taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
4070 taskqueue_free(adapter->reset_tq);
4071
4072 ENA_LOCK_LOCK();
4073 ena_down(adapter);
4074 ena_destroy_device(adapter, true);
4075 ENA_LOCK_UNLOCK();
4076
4077 /* Restore unregistered sysctl queue nodes. */
4078 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues,
4079 adapter->max_num_io_queues);
4080
4081 #ifdef DEV_NETMAP
4082 netmap_detach(adapter->ifp);
4083 #endif /* DEV_NETMAP */
4084
4085 ena_free_stats(adapter);
4086
4087 rc = ena_free_rx_dma_tag(adapter);
4088 if (unlikely(rc != 0))
4089 ena_log(adapter->pdev, WARN,
4090 "Unmapped RX DMA tag associations\n");
4091
4092 rc = ena_free_tx_dma_tag(adapter);
4093 if (unlikely(rc != 0))
4094 ena_log(adapter->pdev, WARN,
4095 "Unmapped TX DMA tag associations\n");
4096
4097 ena_free_irqs(adapter);
4098
4099 ena_free_pci_resources(adapter);
4100
4101 if (adapter->rss_indir != NULL)
4102 free(adapter->rss_indir, M_DEVBUF);
4103
4104 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter)))
4105 ena_com_rss_destroy(ena_dev);
4106
4107 ena_com_delete_host_info(ena_dev);
4108
4109 free(adapter->customer_metrics_array, M_DEVBUF);
4110
4111 ena_com_delete_customer_metrics_buffer(ena_dev);
4112
4113 if_free(adapter->ifp);
4114
4115 free(ena_dev->bus, M_DEVBUF);
4116
4117 free(ena_dev, M_DEVBUF);
4118
4119 return (bus_generic_detach(pdev));
4120 }
4121
4122 /******************************************************************************
4123 ******************************** AENQ Handlers *******************************
4124 *****************************************************************************/
4125 /**
4126 * ena_update_on_link_change:
4127 * Notify the network interface about the change in link status
4128 **/
4129 static void
ena_update_on_link_change(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4130 ena_update_on_link_change(void *adapter_data,
4131 struct ena_admin_aenq_entry *aenq_e)
4132 {
4133 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4134 struct ena_admin_aenq_link_change_desc *aenq_desc;
4135 int status;
4136 if_t ifp;
4137
4138 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
4139 ifp = adapter->ifp;
4140 status = aenq_desc->flags &
4141 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4142
4143 if (status != 0) {
4144 ena_log(adapter->pdev, INFO, "link is UP\n");
4145 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter);
4146 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter))
4147 if_link_state_change(ifp, LINK_STATE_UP);
4148 } else {
4149 ena_log(adapter->pdev, INFO, "link is DOWN\n");
4150 if_link_state_change(ifp, LINK_STATE_DOWN);
4151 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter);
4152 }
4153 }
4154
4155 static void
ena_notification(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4156 ena_notification(void *adapter_data, struct ena_admin_aenq_entry *aenq_e)
4157 {
4158 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4159 struct ena_admin_ena_hw_hints *hints;
4160
4161 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4162 adapter->ena_dev, "Invalid group(%x) expected %x\n",
4163 aenq_e->aenq_common_desc.group, ENA_ADMIN_NOTIFICATION);
4164
4165 switch (aenq_e->aenq_common_desc.syndrome) {
4166 case ENA_ADMIN_UPDATE_HINTS:
4167 hints =
4168 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4);
4169 ena_update_hints(adapter, hints);
4170 break;
4171 default:
4172 ena_log(adapter->pdev, ERR,
4173 "Invalid aenq notification link state %d\n",
4174 aenq_e->aenq_common_desc.syndrome);
4175 }
4176 }
4177
4178 static void
ena_lock_init(void * arg)4179 ena_lock_init(void *arg)
4180 {
4181 ENA_LOCK_INIT();
4182 }
4183 SYSINIT(ena_lock_init, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_init, NULL);
4184
4185 static void
ena_lock_uninit(void * arg)4186 ena_lock_uninit(void *arg)
4187 {
4188 ENA_LOCK_DESTROY();
4189 }
4190 SYSUNINIT(ena_lock_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_uninit, NULL);
4191
4192 /**
4193 * This handler will called for unknown event group or unimplemented handlers
4194 **/
4195 static void
unimplemented_aenq_handler(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4196 unimplemented_aenq_handler(void *adapter_data,
4197 struct ena_admin_aenq_entry *aenq_e)
4198 {
4199 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4200
4201 ena_log(adapter->pdev, ERR,
4202 "Unknown event was received or event with unimplemented handler\n");
4203 }
4204
ena_conf_notification(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4205 static void ena_conf_notification(void *adapter_data,
4206 struct ena_admin_aenq_entry *aenq_e)
4207 {
4208 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4209 struct ena_admin_aenq_conf_notifications_desc *desc;
4210 u64 bitmap, bit;
4211
4212 desc = (struct ena_admin_aenq_conf_notifications_desc *)aenq_e;
4213 bitmap = desc->notifications_bitmap;
4214
4215 if (bitmap == 0) {
4216 ena_log(adapter->pdev, INFO,
4217 "Empty configuration notification bitmap\n");
4218 return;
4219 }
4220
4221 for (bit = ffsll(bitmap); bit != 0; bit = ffsll(bitmap)) {
4222 bit--;
4223 ena_log(adapter->pdev, INFO,
4224 "Sub-optimal configuration notification code: %" PRIu64 " Refer to AWS ENA documentation for additional details and mitigation options.\n",
4225 bit + 1);
4226 // Clear the processed bit
4227 bitmap &= ~(1UL << bit);
4228 }
4229 }
4230
ena_admin_device_request_reset(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4231 static void ena_admin_device_request_reset(void *adapter_data,
4232 struct ena_admin_aenq_entry *aenq_e)
4233 {
4234 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4235 ena_log(adapter->pdev, WARN,
4236 "The device has detected an unhealthy state, reset is requested\n");
4237 ena_trigger_reset(adapter, ENA_REGS_RESET_DEVICE_REQUEST);
4238 }
4239
4240 static struct ena_aenq_handlers aenq_handlers = {
4241 .handlers = {
4242 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4243 [ENA_ADMIN_NOTIFICATION] = ena_notification,
4244 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4245 [ENA_ADMIN_CONF_NOTIFICATIONS] = ena_conf_notification,
4246 [ENA_ADMIN_DEVICE_REQUEST_RESET] = ena_admin_device_request_reset,
4247 },
4248 .unimplemented_handler = unimplemented_aenq_handler
4249 };
4250
4251 /*********************************************************************
4252 * FreeBSD Device Interface Entry Points
4253 *********************************************************************/
4254
4255 static device_method_t ena_methods[] = { /* Device interface */
4256 DEVMETHOD(device_probe, ena_probe),
4257 DEVMETHOD(device_attach, ena_attach),
4258 DEVMETHOD(device_detach, ena_detach), DEVMETHOD_END
4259 };
4260
4261 static driver_t ena_driver = {
4262 "ena",
4263 ena_methods,
4264 sizeof(struct ena_adapter),
4265 };
4266
4267 DRIVER_MODULE(ena, pci, ena_driver, 0, 0);
4268 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array,
4269 nitems(ena_vendor_info_array) - 1);
4270 MODULE_DEPEND(ena, pci, 1, 1, 1);
4271 MODULE_DEPEND(ena, ether, 1, 1, 1);
4272 #ifdef DEV_NETMAP
4273 MODULE_DEPEND(ena, netmap, 1, 1, 1);
4274 #endif /* DEV_NETMAP */
4275
4276 /*********************************************************************/
4277