1 /*-
2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD$
26 */
27
28 #include "en.h"
29 #include <machine/atomic.h>
30
31 void
mlx5e_send_nop(struct mlx5e_sq * sq,u32 ds_cnt,bool notify_hw)32 mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt, bool notify_hw)
33 {
34 u16 pi = sq->pc & sq->wq.sz_m1;
35 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
36
37 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
38
39 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
40 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
41 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
42
43 sq->mbuf[pi].mbuf = NULL;
44 sq->mbuf[pi].num_bytes = 0;
45 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
46 sq->pc += sq->mbuf[pi].num_wqebbs;
47 if (notify_hw)
48 mlx5e_tx_notify_hw(sq, wqe, 0);
49 }
50
51 #if (__FreeBSD_version >= 1100000)
52 static uint32_t mlx5e_hash_value;
53
54 static void
mlx5e_hash_init(void * arg)55 mlx5e_hash_init(void *arg)
56 {
57 mlx5e_hash_value = m_ether_tcpip_hash_init();
58 }
59
60 /* Make kernel call mlx5e_hash_init after the random stack finished initializing */
61 SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL);
62 #endif
63
64 static struct mlx5e_sq *
mlx5e_select_queue(struct ifnet * ifp,struct mbuf * mb)65 mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb)
66 {
67 struct mlx5e_priv *priv = ifp->if_softc;
68 u32 ch;
69 u32 tc;
70
71 /* check if channels are successfully opened */
72 if (unlikely(priv->channel == NULL))
73 return (NULL);
74
75 /* obtain VLAN information if present */
76 if (mb->m_flags & M_VLANTAG) {
77 tc = (mb->m_pkthdr.ether_vtag >> 13);
78 if (tc >= priv->num_tc)
79 tc = priv->default_vlan_prio;
80 } else {
81 tc = priv->default_vlan_prio;
82 }
83
84 ch = priv->params.num_channels;
85
86 /* check if flowid is set */
87 if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
88 #ifdef RSS
89 u32 temp;
90
91 if (rss_hash2bucket(mb->m_pkthdr.flowid,
92 M_HASHTYPE_GET(mb), &temp) == 0)
93 ch = temp % ch;
94 else
95 #endif
96 ch = (mb->m_pkthdr.flowid % 128) % ch;
97 } else {
98 #if (__FreeBSD_version >= 1100000)
99 ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |
100 MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch;
101 #else
102 /*
103 * m_ether_tcpip_hash not present in stable, so just
104 * throw unhashed mbufs on queue 0
105 */
106 ch = 0;
107 #endif
108 }
109
110 /* check if channel is allocated */
111 if (unlikely(priv->channel[ch] == NULL))
112 return (NULL);
113
114 return (&priv->channel[ch]->sq[tc]);
115 }
116
117 static inline u16
mlx5e_get_inline_hdr_size(struct mlx5e_sq * sq,struct mbuf * mb)118 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb)
119 {
120 return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len));
121 }
122
123 static int
mlx5e_get_header_size(struct mbuf * mb)124 mlx5e_get_header_size(struct mbuf *mb)
125 {
126 struct ether_vlan_header *eh;
127 struct tcphdr *th;
128 struct ip *ip;
129 int ip_hlen, tcp_hlen;
130 struct ip6_hdr *ip6;
131 uint16_t eth_type;
132 int eth_hdr_len;
133
134 eh = mtod(mb, struct ether_vlan_header *);
135 if (mb->m_len < ETHER_HDR_LEN)
136 return (0);
137 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
138 eth_type = ntohs(eh->evl_proto);
139 eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
140 } else {
141 eth_type = ntohs(eh->evl_encap_proto);
142 eth_hdr_len = ETHER_HDR_LEN;
143 }
144 if (mb->m_len < eth_hdr_len)
145 return (0);
146 switch (eth_type) {
147 case ETHERTYPE_IP:
148 ip = (struct ip *)(mb->m_data + eth_hdr_len);
149 if (mb->m_len < eth_hdr_len + sizeof(*ip))
150 return (0);
151 if (ip->ip_p != IPPROTO_TCP)
152 return (0);
153 ip_hlen = ip->ip_hl << 2;
154 eth_hdr_len += ip_hlen;
155 break;
156 case ETHERTYPE_IPV6:
157 ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len);
158 if (mb->m_len < eth_hdr_len + sizeof(*ip6))
159 return (0);
160 if (ip6->ip6_nxt != IPPROTO_TCP)
161 return (0);
162 eth_hdr_len += sizeof(*ip6);
163 break;
164 default:
165 return (0);
166 }
167 if (mb->m_len < eth_hdr_len + sizeof(*th))
168 return (0);
169 th = (struct tcphdr *)(mb->m_data + eth_hdr_len);
170 tcp_hlen = th->th_off << 2;
171 eth_hdr_len += tcp_hlen;
172 if (mb->m_len < eth_hdr_len)
173 return (0);
174 return (eth_hdr_len);
175 }
176
177 /*
178 * The return value is not going back to the stack because of
179 * the drbr
180 */
181 static int
mlx5e_sq_xmit(struct mlx5e_sq * sq,struct mbuf ** mbp)182 mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
183 {
184 bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS];
185 struct mlx5_wqe_data_seg *dseg;
186 struct mlx5e_tx_wqe *wqe;
187 struct ifnet *ifp;
188 int nsegs;
189 int err;
190 int x;
191 struct mbuf *mb = *mbp;
192 u16 ds_cnt;
193 u16 ihs;
194 u16 pi;
195 u8 opcode;
196
197 /*
198 * Return ENOBUFS if the queue is full, this may trigger reinsertion
199 * of the mbuf into the drbr (see mlx5e_xmit_locked)
200 */
201 if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) {
202 return (ENOBUFS);
203 }
204
205 /* Align SQ edge with NOPs to avoid WQE wrap around */
206 pi = ((~sq->pc) & sq->wq.sz_m1);
207 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
208 /* Send one multi NOP message instead of many */
209 mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS, false);
210 pi = ((~sq->pc) & sq->wq.sz_m1);
211 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
212 m_freem(mb);
213 return (ENOMEM);
214 }
215 }
216
217 /* Setup local variables */
218 pi = sq->pc & sq->wq.sz_m1;
219 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
220 ifp = sq->channel->ifp;
221
222 memset(wqe, 0, sizeof(*wqe));
223
224 /* Send a copy of the frame to the BPF listener, if any */
225 if (ifp != NULL && ifp->if_bpf != NULL)
226 ETHER_BPF_MTAP(ifp, mb);
227
228 if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) {
229 wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM;
230 }
231 if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) {
232 wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM;
233 }
234 if (wqe->eth.cs_flags == 0) {
235 sq->stats.csum_offload_none++;
236 }
237 if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
238 u32 payload_len;
239 u32 mss = mb->m_pkthdr.tso_segsz;
240 u32 num_pkts;
241
242 wqe->eth.mss = cpu_to_be16(mss);
243 opcode = MLX5_OPCODE_LSO;
244 ihs = mlx5e_get_header_size(mb);
245 payload_len = mb->m_pkthdr.len - ihs;
246 if (payload_len == 0)
247 num_pkts = 1;
248 else
249 num_pkts = DIV_ROUND_UP(payload_len, mss);
250 sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs);
251
252 sq->stats.tso_packets++;
253 sq->stats.tso_bytes += payload_len;
254 } else {
255 opcode = MLX5_OPCODE_SEND;
256 ihs = mlx5e_get_inline_hdr_size(sq, mb);
257 sq->mbuf[pi].num_bytes = max_t (unsigned int,
258 mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
259 }
260 if (mb->m_flags & M_VLANTAG) {
261 struct ether_vlan_header *eh =
262 (struct ether_vlan_header *)wqe->eth.inline_hdr_start;
263
264 /* Range checks */
265 if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))
266 ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN);
267 else if (ihs < ETHER_HDR_LEN) {
268 err = EINVAL;
269 goto tx_drop;
270 }
271 m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
272 m_adj(mb, ETHER_HDR_LEN);
273 /* Insert 4 bytes VLAN tag into data stream */
274 eh->evl_proto = eh->evl_encap_proto;
275 eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
276 eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
277 /* Copy rest of header data, if any */
278 m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1));
279 m_adj(mb, ihs - ETHER_HDR_LEN);
280 /* Extend header by 4 bytes */
281 ihs += ETHER_VLAN_ENCAP_LEN;
282 } else {
283 m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start);
284 m_adj(mb, ihs);
285 }
286
287 wqe->eth.inline_hdr_sz = cpu_to_be16(ihs);
288
289 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
290 if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) {
291 ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start),
292 MLX5_SEND_WQE_DS);
293 }
294 dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt;
295
296 /* Trim off empty mbufs */
297 while (mb->m_len == 0) {
298 mb = m_free(mb);
299 /* Check if all data has been inlined */
300 if (mb == NULL)
301 goto skip_dma;
302 }
303
304 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
305 mb, segs, &nsegs, BUS_DMA_NOWAIT);
306 if (err == EFBIG) {
307 /*
308 * Update *mbp before defrag in case it was trimmed in the
309 * loop above
310 */
311 *mbp = mb;
312 /* Update statistics */
313 sq->stats.defragged++;
314 /* Too many mbuf fragments */
315 mb = m_defrag(*mbp, M_NOWAIT);
316 if (mb == NULL) {
317 mb = *mbp;
318 goto tx_drop;
319 }
320 /* Try again */
321 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
322 mb, segs, &nsegs, BUS_DMA_NOWAIT);
323 }
324 /* Catch errors */
325 if (err != 0) {
326 goto tx_drop;
327 }
328 *mbp = mb;
329
330 for (x = 0; x != nsegs; x++) {
331 if (segs[x].ds_len == 0)
332 continue;
333 dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr);
334 dseg->lkey = sq->mkey_be;
335 dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len);
336 dseg++;
337 }
338 skip_dma:
339 ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl));
340
341 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
342 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
343 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
344
345 /* Store pointer to mbuf */
346 sq->mbuf[pi].mbuf = mb;
347 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
348 sq->pc += sq->mbuf[pi].num_wqebbs;
349
350 /* Make sure all mbuf data is written to RAM */
351 if (mb != NULL)
352 bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE);
353
354 mlx5e_tx_notify_hw(sq, wqe, 0);
355
356 sq->stats.packets++;
357 return (0);
358
359 tx_drop:
360 sq->stats.dropped++;
361 *mbp = NULL;
362 m_freem(mb);
363 return err;
364 }
365
366 static void
mlx5e_poll_tx_cq(struct mlx5e_sq * sq,int budget)367 mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
368 {
369 u16 sqcc;
370
371 /*
372 * sq->cc must be updated only after mlx5_cqwq_update_db_record(),
373 * otherwise a cq overrun may occur
374 */
375 sqcc = sq->cc;
376
377 while (budget--) {
378 struct mlx5_cqe64 *cqe;
379 struct mbuf *mb;
380 u16 ci;
381
382 cqe = mlx5e_get_cqe(&sq->cq);
383 if (!cqe)
384 break;
385
386 mlx5_cqwq_pop(&sq->cq.wq);
387
388 ci = sqcc & sq->wq.sz_m1;
389 mb = sq->mbuf[ci].mbuf;
390 sq->mbuf[ci].mbuf = NULL; /* Safety clear */
391
392 if (mb == NULL) {
393 if (sq->mbuf[ci].num_bytes == 0) {
394 /* NOP */
395 sq->stats.nop++;
396 }
397 } else {
398 bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
399 BUS_DMASYNC_POSTWRITE);
400 bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map);
401
402 /* Free transmitted mbuf */
403 m_freem(mb);
404 }
405 sqcc += sq->mbuf[ci].num_wqebbs;
406 }
407
408 mlx5_cqwq_update_db_record(&sq->cq.wq);
409
410 /* Ensure cq space is freed before enabling more cqes */
411 wmb();
412
413 sq->cc = sqcc;
414
415 if (atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY))
416 taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
417 }
418
419 static int
mlx5e_xmit_locked(struct ifnet * ifp,struct mlx5e_sq * sq,struct mbuf * mb)420 mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
421 {
422 struct mbuf *next;
423 int err = 0;
424
425 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
426 if (mb)
427 err = drbr_enqueue(ifp, sq->br, mb);
428 return (err);
429 }
430
431 if (mb != NULL)
432 /*
433 * If we can't insert mbuf into drbr, try to xmit anyway.
434 * We keep the error we got so we could return that after xmit.
435 */
436 err = drbr_enqueue(ifp, sq->br, mb);
437
438 /* Process the queue */
439 while ((next = drbr_peek(ifp, sq->br)) != NULL) {
440 if (mlx5e_sq_xmit(sq, &next) != 0) {
441 if (next == NULL) {
442 drbr_advance(ifp, sq->br);
443 } else {
444 drbr_putback(ifp, sq->br, next);
445 atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL);
446 }
447 break;
448 }
449 drbr_advance(ifp, sq->br);
450 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
451 break;
452 }
453 return (err);
454 }
455
456 int
mlx5e_xmit(struct ifnet * ifp,struct mbuf * mb)457 mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb)
458 {
459 struct mlx5e_sq *sq;
460 int ret;
461
462 sq = mlx5e_select_queue(ifp, mb);
463 if (unlikely(sq == NULL)) {
464 /* Invalid send queue */
465 m_freem(mb);
466 return (ENXIO);
467 }
468 if (mtx_trylock(&sq->lock)) {
469 ret = mlx5e_xmit_locked(ifp, sq, mb);
470 mtx_unlock(&sq->lock);
471 } else {
472 ret = drbr_enqueue(ifp, sq->br, mb);
473 taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
474 }
475
476 return (ret);
477 }
478
479 void
mlx5e_tx_cq_comp(struct mlx5_core_cq * mcq)480 mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq)
481 {
482 struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq);
483
484 mtx_lock(&sq->comp_lock);
485 mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX);
486 mlx5e_cq_arm(&sq->cq);
487 mtx_unlock(&sq->comp_lock);
488 }
489
490 void
mlx5e_tx_que(void * context,int pending)491 mlx5e_tx_que(void *context, int pending)
492 {
493 struct mlx5e_sq *sq = context;
494 struct ifnet *ifp = sq->channel->ifp;
495
496 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
497 mtx_lock(&sq->lock);
498 if (!drbr_empty(ifp, sq->br))
499 mlx5e_xmit_locked(ifp, sq, NULL);
500 mtx_unlock(&sq->lock);
501 }
502 }
503