55     struct mbuf *);
56 static struct mbuf *ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
59     struct mbuf *);
60 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool);
62     struct mbuf **mbuf);
63 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
130 ena_mq_start(if_t ifp, struct mbuf *m)  in ena_mq_start()
219 	if (tx_ring->tx_buffer_info[*req_id].mbuf != NULL)  in ena_get_tx_req_id()
269 		struct mbuf *mbuf;  in ena_tx_cleanup()  local
277 		mbuf = tx_info->mbuf;  in ena_tx_cleanup()
279 		tx_info->mbuf = NULL;  in ena_tx_cleanup()
287 		    tx_ring->qid, mbuf);  in ena_tx_cleanup()
289 		m_freem(mbuf);  in ena_tx_cleanup()
350     struct mbuf *mbuf)  in ena_rx_hash_mbuf()  argument
355 		mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;  in ena_rx_hash_mbuf()
365 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);  in ena_rx_hash_mbuf()
372 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);  in ena_rx_hash_mbuf()
380 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);  in ena_rx_hash_mbuf()
383 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);  in ena_rx_hash_mbuf()
386 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);  in ena_rx_hash_mbuf()
392 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);  in ena_rx_hash_mbuf()
395 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);  in ena_rx_hash_mbuf()
398 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);  in ena_rx_hash_mbuf()
402 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);  in ena_rx_hash_mbuf()
405 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);  in ena_rx_hash_mbuf()
408 		mbuf->m_pkthdr.flowid = rx_ring->qid;  in ena_rx_hash_mbuf()
409 		M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);  in ena_rx_hash_mbuf()
421 static struct mbuf *
425 	struct mbuf *mbuf;  in ena_rx_mbuf()  local
439 	if (unlikely(rx_info->mbuf == NULL)) {  in ena_rx_mbuf()
445 	    rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);  in ena_rx_mbuf()
449 	mbuf = rx_info->mbuf;  in ena_rx_mbuf()
450 	mbuf->m_flags |= M_PKTHDR;  in ena_rx_mbuf()
451 	mbuf->m_pkthdr.len = len;  in ena_rx_mbuf()
452 	mbuf->m_len = len;  in ena_rx_mbuf()
454 	mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset);  in ena_rx_mbuf()
456 	mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;  in ena_rx_mbuf()
459 	ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);  in ena_rx_mbuf()
461 	ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf,  in ena_rx_mbuf()
462 	    mbuf->m_flags, mbuf->m_pkthdr.len);  in ena_rx_mbuf()
467 	rx_info->mbuf = NULL;  in ena_rx_mbuf()
481 		if (unlikely(rx_info->mbuf == NULL)) {  in ena_rx_mbuf()
493 			m_freem(mbuf);  in ena_rx_mbuf()
499 		if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {  in ena_rx_mbuf()
502 			    mbuf);  in ena_rx_mbuf()
506 		    mbuf->m_pkthdr.len);  in ena_rx_mbuf()
510 		m_freem(rx_info->mbuf);  in ena_rx_mbuf()
511 		rx_info->mbuf = NULL;  in ena_rx_mbuf()
519 	return (mbuf);  in ena_rx_mbuf()
527     struct mbuf *mbuf)  in ena_rx_checksum()  argument
535 		mbuf->m_pkthdr.csum_flags = 0;  in ena_rx_checksum()
546 			mbuf->m_pkthdr.csum_flags = 0;  in ena_rx_checksum()
550 			mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;  in ena_rx_checksum()
551 			mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;  in ena_rx_checksum()
566 	struct mbuf *mbuf;  in ena_rx_cleanup()  local
632 		mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, &ena_rx_ctx,  in ena_rx_cleanup()
637 		if (unlikely(mbuf == NULL)) {  in ena_rx_cleanup()
649 			ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);  in ena_rx_cleanup()
654 		    mbuf->m_pkthdr.len);  in ena_rx_cleanup()
656 		    mbuf->m_pkthdr.len);  in ena_rx_cleanup()
664 		    ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&  in ena_rx_cleanup()
673 			    (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))  in ena_rx_cleanup()
678 			    "calling if_input() with mbuf %p\n", mbuf);  in ena_rx_cleanup()
679 			(*ifp->if_input)(ifp, mbuf);  in ena_rx_cleanup()
706 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf,  in ena_tx_csum()  argument
711 	struct mbuf *mbuf_next;  in ena_tx_csum()
724 	mss = mbuf->m_pkthdr.tso_segsz;  in ena_tx_csum()
729 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)  in ena_tx_csum()
732 	if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)  in ena_tx_csum()
735 	if ((mbuf->m_pkthdr.csum_flags & CSUM6_OFFLOAD) != 0)  in ena_tx_csum()
749 	eh = mtod(mbuf, struct ether_vlan_header *);  in ena_tx_csum()
758 	mbuf_next = m_getptr(mbuf, ehdrlen, &offset);  in ena_tx_csum()
771 		iphlen = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &ipproto);  in ena_tx_csum()
781 	mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset);  in ena_tx_csum()
784 	if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {  in ena_tx_csum()
787 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {  in ena_tx_csum()
794 		if ((mbuf->m_pkthdr.csum_flags &  in ena_tx_csum()
801 		if ((mbuf->m_pkthdr.csum_flags &  in ena_tx_csum()
818 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)  in ena_check_and_collapse_mbuf()  argument
821 	struct mbuf *collapsed_mbuf;  in ena_check_and_collapse_mbuf()
825 	num_frags = ena_mbuf_count(*mbuf);  in ena_check_and_collapse_mbuf()
832 	    ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size))  in ena_check_and_collapse_mbuf()
837 	collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,  in ena_check_and_collapse_mbuf()
845 	*mbuf = collapsed_mbuf;  in ena_check_and_collapse_mbuf()
852     struct mbuf *mbuf, void **push_hdr, u16 *header_len)  in ena_tx_map_mbuf()  argument
862 	mbuf_head_len = mbuf->m_len;  in ena_tx_map_mbuf()
863 	tx_info->mbuf = mbuf;  in ena_tx_map_mbuf()
871 	    tx_info->dmamap, mbuf, segs, &nsegs, BUS_DMA_NOWAIT);  in ena_tx_map_mbuf()
889 		*header_len = min_t(uint32_t, mbuf->m_pkthdr.len,  in ena_tx_map_mbuf()
894 			*push_hdr = mbuf->m_data;  in ena_tx_map_mbuf()
900 			m_copydata(mbuf, 0, *header_len,  in ena_tx_map_mbuf()
909 		    mbuf, *push_hdr, *header_len);  in ena_tx_map_mbuf()
912 		if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) {  in ena_tx_map_mbuf()
961 	tx_info->mbuf = NULL;  in ena_tx_map_mbuf()
966 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)  in ena_xmit_mbuf()  argument
988 	rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);  in ena_xmit_mbuf()
995 	ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);  in ena_xmit_mbuf()
1002 	ENA_WARN(tx_info->mbuf != NULL, adapter->ena_dev,  in ena_xmit_mbuf()
1005 	rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);  in ena_xmit_mbuf()
1018 	ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching);  in ena_xmit_mbuf()
1046 	    (*mbuf)->m_pkthdr.len);  in ena_xmit_mbuf()
1050 	    (*mbuf)->m_pkthdr.len);  in ena_xmit_mbuf()
1093 	tx_info->mbuf = NULL;  in ena_xmit_mbuf()
1102 	struct mbuf *mbuf;  in ena_start_xmit()  local
1114 	while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {  in ena_start_xmit()
1117 		    mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);  in ena_start_xmit()
1120 			drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1124 		if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {  in ena_start_xmit()
1126 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1128 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);  in ena_start_xmit()
1130 				m_freem(mbuf);  in ena_start_xmit()
1144 		BPF_MTAP(adapter->ifp, mbuf);  in ena_start_xmit()