1 /*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33
34 #ifdef DEV_NETMAP
35 #include <sys/param.h>
36 #include <sys/bus.h>
37 #include <sys/eventhandler.h>
38 #include <sys/lock.h>
39 #include <sys/mbuf.h>
40 #include <sys/module.h>
41 #include <sys/selinfo.h>
42 #include <sys/socket.h>
43 #include <sys/sockio.h>
44 #include <machine/bus.h>
45 #include <net/ethernet.h>
46 #include <net/if.h>
47 #include <net/if_media.h>
48 #include <net/if_var.h>
49 #include <net/if_clone.h>
50 #include <net/if_types.h>
51 #include <net/netmap.h>
52 #include <dev/netmap/netmap_kern.h>
53
54 #include "common/common.h"
55 #include "common/t4_regs.h"
56 #include "common/t4_regs_values.h"
57
58 extern int fl_pad; /* XXXNM */
59 extern int spg_len; /* XXXNM */
60 extern int fl_pktshift; /* XXXNM */
61
62 SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0, "cxgbe netmap parameters");
63
64 /*
65 * 0 = normal netmap rx
66 * 1 = black hole
67 * 2 = supermassive black hole (buffer packing enabled)
68 */
69 int black_hole = 0;
70 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_black_hole, CTLFLAG_RDTUN, &black_hole, 0,
71 "Sink incoming packets.");
72
73 int rx_ndesc = 256;
74 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_rx_ndesc, CTLFLAG_RWTUN,
75 &rx_ndesc, 0, "# of rx descriptors after which the hw cidx is updated.");
76
77 int holdoff_tmr_idx = 2;
78 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_holdoff_tmr_idx, CTLFLAG_RWTUN,
79 &holdoff_tmr_idx, 0, "Holdoff timer index for netmap rx queues.");
80
81 /*
82 * Congestion drops.
83 * -1: no congestion feedback (not recommended).
84 * 0: backpressure the channel instead of dropping packets right away.
85 * 1: no backpressure, drop packets for the congested queue immediately.
86 */
87 static int nm_cong_drop = 1;
88 TUNABLE_INT("hw.cxgbe.nm_cong_drop", &nm_cong_drop);
89
90 /* netmap ifnet routines */
91 static void cxgbe_nm_init(void *);
92 static int cxgbe_nm_ioctl(struct ifnet *, unsigned long, caddr_t);
93 static int cxgbe_nm_transmit(struct ifnet *, struct mbuf *);
94 static void cxgbe_nm_qflush(struct ifnet *);
95
96 static int cxgbe_nm_init_synchronized(struct vi_info *);
97 static int cxgbe_nm_uninit_synchronized(struct vi_info *);
98
99 /* T4 netmap VI (ncxgbe) interface */
100 static int ncxgbe_probe(device_t);
101 static int ncxgbe_attach(device_t);
102 static int ncxgbe_detach(device_t);
103 static device_method_t ncxgbe_methods[] = {
104 DEVMETHOD(device_probe, ncxgbe_probe),
105 DEVMETHOD(device_attach, ncxgbe_attach),
106 DEVMETHOD(device_detach, ncxgbe_detach),
107 { 0, 0 }
108 };
109 static driver_t ncxgbe_driver = {
110 "ncxgbe",
111 ncxgbe_methods,
112 sizeof(struct vi_info)
113 };
114
115 /* T5 netmap VI (ncxl) interface */
116 static driver_t ncxl_driver = {
117 "ncxl",
118 ncxgbe_methods,
119 sizeof(struct vi_info)
120 };
121
122 static void
cxgbe_nm_init(void * arg)123 cxgbe_nm_init(void *arg)
124 {
125 struct vi_info *vi = arg;
126 struct adapter *sc = vi->pi->adapter;
127
128 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4nminit") != 0)
129 return;
130 cxgbe_nm_init_synchronized(vi);
131 end_synchronized_op(sc, 0);
132
133 return;
134 }
135
136 static int
cxgbe_nm_init_synchronized(struct vi_info * vi)137 cxgbe_nm_init_synchronized(struct vi_info *vi)
138 {
139 struct adapter *sc = vi->pi->adapter;
140 struct ifnet *ifp = vi->ifp;
141 int rc = 0;
142
143 ASSERT_SYNCHRONIZED_OP(sc);
144
145 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
146 return (0); /* already running */
147
148 if (!(sc->flags & FULL_INIT_DONE) &&
149 ((rc = adapter_full_init(sc)) != 0))
150 return (rc); /* error message displayed already */
151
152 if (!(vi->flags & VI_INIT_DONE) &&
153 ((rc = vi_full_init(vi)) != 0))
154 return (rc); /* error message displayed already */
155
156 rc = update_mac_settings(ifp, XGMAC_ALL);
157 if (rc)
158 return (rc); /* error message displayed already */
159
160 ifp->if_drv_flags |= IFF_DRV_RUNNING;
161 callout_reset(&vi->tick, hz, vi_tick, vi);
162
163 return (rc);
164 }
165
166 static int
cxgbe_nm_uninit_synchronized(struct vi_info * vi)167 cxgbe_nm_uninit_synchronized(struct vi_info *vi)
168 {
169 #ifdef INVARIANTS
170 struct adapter *sc = vi->pi->adapter;
171 #endif
172 struct ifnet *ifp = vi->ifp;
173
174 ASSERT_SYNCHRONIZED_OP(sc);
175
176 callout_stop(&vi->tick);
177 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
178
179 return (0);
180 }
181
182 static int
cxgbe_nm_ioctl(struct ifnet * ifp,unsigned long cmd,caddr_t data)183 cxgbe_nm_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
184 {
185 int rc = 0, mtu, flags;
186 struct vi_info *vi = ifp->if_softc;
187 struct adapter *sc = vi->pi->adapter;
188 struct ifreq *ifr = (struct ifreq *)data;
189 uint32_t mask;
190
191 MPASS(vi->ifp == ifp);
192
193 switch (cmd) {
194 case SIOCSIFMTU:
195 mtu = ifr->ifr_mtu;
196 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
197 return (EINVAL);
198
199 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4nmtu");
200 if (rc)
201 return (rc);
202 ifp->if_mtu = mtu;
203 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
204 rc = update_mac_settings(ifp, XGMAC_MTU);
205 end_synchronized_op(sc, 0);
206 break;
207
208 case SIOCSIFFLAGS:
209 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4nflg");
210 if (rc)
211 return (rc);
212
213 if (ifp->if_flags & IFF_UP) {
214 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
215 flags = vi->if_flags;
216 if ((ifp->if_flags ^ flags) &
217 (IFF_PROMISC | IFF_ALLMULTI)) {
218 rc = update_mac_settings(ifp,
219 XGMAC_PROMISC | XGMAC_ALLMULTI);
220 }
221 } else
222 rc = cxgbe_nm_init_synchronized(vi);
223 vi->if_flags = ifp->if_flags;
224 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
225 rc = cxgbe_nm_uninit_synchronized(vi);
226 end_synchronized_op(sc, 0);
227 break;
228
229 case SIOCADDMULTI:
230 case SIOCDELMULTI: /* these two are called with a mutex held :-( */
231 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4nmulti");
232 if (rc)
233 return (rc);
234 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
235 rc = update_mac_settings(ifp, XGMAC_MCADDRS);
236 end_synchronized_op(sc, LOCK_HELD);
237 break;
238
239 case SIOCSIFCAP:
240 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
241 if (mask & IFCAP_TXCSUM) {
242 ifp->if_capenable ^= IFCAP_TXCSUM;
243 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
244 }
245 if (mask & IFCAP_TXCSUM_IPV6) {
246 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
247 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
248 }
249 if (mask & IFCAP_RXCSUM)
250 ifp->if_capenable ^= IFCAP_RXCSUM;
251 if (mask & IFCAP_RXCSUM_IPV6)
252 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
253 break;
254
255 case SIOCSIFMEDIA:
256 case SIOCGIFMEDIA:
257 ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
258 break;
259
260 default:
261 rc = ether_ioctl(ifp, cmd, data);
262 }
263
264 return (rc);
265 }
266
267 static int
cxgbe_nm_transmit(struct ifnet * ifp,struct mbuf * m)268 cxgbe_nm_transmit(struct ifnet *ifp, struct mbuf *m)
269 {
270
271 m_freem(m);
272 return (0);
273 }
274
275 static void
cxgbe_nm_qflush(struct ifnet * ifp)276 cxgbe_nm_qflush(struct ifnet *ifp)
277 {
278
279 return;
280 }
281
282 static int
alloc_nm_rxq_hwq(struct vi_info * vi,struct sge_nm_rxq * nm_rxq,int cong)283 alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int cong)
284 {
285 int rc, cntxt_id, i;
286 __be32 v;
287 struct adapter *sc = vi->pi->adapter;
288 struct netmap_adapter *na = NA(vi->ifp);
289 struct fw_iq_cmd c;
290
291 MPASS(na != NULL);
292 MPASS(nm_rxq->iq_desc != NULL);
293 MPASS(nm_rxq->fl_desc != NULL);
294
295 bzero(nm_rxq->iq_desc, vi->qsize_rxq * IQ_ESIZE);
296 bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + spg_len);
297
298 bzero(&c, sizeof(c));
299 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
300 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
301 V_FW_IQ_CMD_VFN(0));
302 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
303 FW_LEN16(c));
304 if (vi->flags & INTR_RXQ) {
305 KASSERT(nm_rxq->intr_idx < sc->intr_count,
306 ("%s: invalid direct intr_idx %d", __func__,
307 nm_rxq->intr_idx));
308 v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);
309 } else {
310 CXGBE_UNIMPLEMENTED(__func__); /* XXXNM: needs review */
311 v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx) |
312 F_FW_IQ_CMD_IQANDST;
313 }
314 c.type_to_iqandstindex = htobe32(v |
315 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
316 V_FW_IQ_CMD_VIID(vi->viid) |
317 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
318 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(vi->pi->tx_chan) |
319 F_FW_IQ_CMD_IQGTSMODE |
320 V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
321 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
322 c.iqsize = htobe16(vi->qsize_rxq);
323 c.iqaddr = htobe64(nm_rxq->iq_ba);
324 if (cong >= 0) {
325 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN |
326 V_FW_IQ_CMD_FL0CNGCHMAP(cong) | F_FW_IQ_CMD_FL0CONGCIF |
327 F_FW_IQ_CMD_FL0CONGEN);
328 }
329 c.iqns_to_fl0congen |=
330 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
331 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
332 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
333 (black_hole == 2 ? F_FW_IQ_CMD_FL0PACKEN : 0));
334 c.fl0dcaen_to_fl0cidxfthresh =
335 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) |
336 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
337 c.fl0size = htobe16(na->num_rx_desc / 8 + spg_len / EQ_ESIZE);
338 c.fl0addr = htobe64(nm_rxq->fl_ba);
339
340 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
341 if (rc != 0) {
342 device_printf(sc->dev,
343 "failed to create netmap ingress queue: %d\n", rc);
344 return (rc);
345 }
346
347 nm_rxq->iq_cidx = 0;
348 MPASS(nm_rxq->iq_sidx == vi->qsize_rxq - spg_len / IQ_ESIZE);
349 nm_rxq->iq_gen = F_RSPD_GEN;
350 nm_rxq->iq_cntxt_id = be16toh(c.iqid);
351 nm_rxq->iq_abs_id = be16toh(c.physiqid);
352 cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
353 if (cntxt_id >= sc->sge.niq) {
354 panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
355 __func__, cntxt_id, sc->sge.niq - 1);
356 }
357 sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;
358
359 nm_rxq->fl_cntxt_id = be16toh(c.fl0id);
360 nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
361 MPASS(nm_rxq->fl_sidx == na->num_rx_desc);
362 cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;
363 if (cntxt_id >= sc->sge.neq) {
364 panic("%s: nm_rxq->fl_cntxt_id (%d) more than the max (%d)",
365 __func__, cntxt_id, sc->sge.neq - 1);
366 }
367 sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
368
369 nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
370 if (is_t5(sc))
371 nm_rxq->fl_db_val |= F_DBTYPE;
372
373 if (is_t5(sc) && cong >= 0) {
374 uint32_t param, val;
375
376 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
377 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
378 V_FW_PARAMS_PARAM_YZ(nm_rxq->iq_cntxt_id);
379 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
380 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
381 V_FW_PARAMS_PARAM_YZ(nm_rxq->iq_cntxt_id);
382 if (cong == 0)
383 val = 1 << 19;
384 else {
385 val = 2 << 19;
386 for (i = 0; i < 4; i++) {
387 if (cong & (1 << i))
388 val |= 1 << (i << 2);
389 }
390 }
391
392 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
393 if (rc != 0) {
394 /* report error but carry on */
395 device_printf(sc->dev,
396 "failed to set congestion manager context for "
397 "ingress queue %d: %d\n", nm_rxq->iq_cntxt_id, rc);
398 }
399 }
400
401 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
402 V_INGRESSQID(nm_rxq->iq_cntxt_id) |
403 V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
404
405 return (rc);
406 }
407
408 static int
free_nm_rxq_hwq(struct vi_info * vi,struct sge_nm_rxq * nm_rxq)409 free_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)
410 {
411 struct adapter *sc = vi->pi->adapter;
412 int rc;
413
414 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
415 nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);
416 if (rc != 0)
417 device_printf(sc->dev, "%s: failed for iq %d, fl %d: %d\n",
418 __func__, nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, rc);
419 return (rc);
420 }
421
422 static int
alloc_nm_txq_hwq(struct vi_info * vi,struct sge_nm_txq * nm_txq)423 alloc_nm_txq_hwq(struct vi_info *vi, struct sge_nm_txq *nm_txq)
424 {
425 int rc, cntxt_id;
426 size_t len;
427 struct adapter *sc = vi->pi->adapter;
428 struct netmap_adapter *na = NA(vi->ifp);
429 struct fw_eq_eth_cmd c;
430
431 MPASS(na != NULL);
432 MPASS(nm_txq->desc != NULL);
433
434 len = na->num_tx_desc * EQ_ESIZE + spg_len;
435 bzero(nm_txq->desc, len);
436
437 bzero(&c, sizeof(c));
438 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
439 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
440 V_FW_EQ_ETH_CMD_VFN(0));
441 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
442 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
443 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
444 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
445 c.fetchszm_to_iqid =
446 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
447 V_FW_EQ_ETH_CMD_PCIECHN(vi->pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
448 V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
449 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
450 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
451 V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
452 c.eqaddr = htobe64(nm_txq->ba);
453
454 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
455 if (rc != 0) {
456 device_printf(vi->dev,
457 "failed to create netmap egress queue: %d\n", rc);
458 return (rc);
459 }
460
461 nm_txq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
462 cntxt_id = nm_txq->cntxt_id - sc->sge.eq_start;
463 if (cntxt_id >= sc->sge.neq)
464 panic("%s: nm_txq->cntxt_id (%d) more than the max (%d)", __func__,
465 cntxt_id, sc->sge.neq - 1);
466 sc->sge.eqmap[cntxt_id] = (void *)nm_txq;
467
468 nm_txq->pidx = nm_txq->cidx = 0;
469 MPASS(nm_txq->sidx == na->num_tx_desc);
470 nm_txq->equiqidx = nm_txq->equeqidx = nm_txq->dbidx = 0;
471
472 nm_txq->doorbells = sc->doorbells;
473 if (isset(&nm_txq->doorbells, DOORBELL_UDB) ||
474 isset(&nm_txq->doorbells, DOORBELL_UDBWC) ||
475 isset(&nm_txq->doorbells, DOORBELL_WCWR)) {
476 uint32_t s_qpp = sc->sge.eq_s_qpp;
477 uint32_t mask = (1 << s_qpp) - 1;
478 volatile uint8_t *udb;
479
480 udb = sc->udbs_base + UDBS_DB_OFFSET;
481 udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;
482 nm_txq->udb_qid = nm_txq->cntxt_id & mask;
483 if (nm_txq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
484 clrbit(&nm_txq->doorbells, DOORBELL_WCWR);
485 else {
486 udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;
487 nm_txq->udb_qid = 0;
488 }
489 nm_txq->udb = (volatile void *)udb;
490 }
491
492 return (rc);
493 }
494
495 static int
free_nm_txq_hwq(struct vi_info * vi,struct sge_nm_txq * nm_txq)496 free_nm_txq_hwq(struct vi_info *vi, struct sge_nm_txq *nm_txq)
497 {
498 struct adapter *sc = vi->pi->adapter;
499 int rc;
500
501 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, nm_txq->cntxt_id);
502 if (rc != 0)
503 device_printf(sc->dev, "%s: failed for eq %d: %d\n", __func__,
504 nm_txq->cntxt_id, rc);
505 return (rc);
506 }
507
508 static int
cxgbe_netmap_on(struct adapter * sc,struct vi_info * vi,struct ifnet * ifp,struct netmap_adapter * na)509 cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp,
510 struct netmap_adapter *na)
511 {
512 struct netmap_slot *slot;
513 struct sge_nm_rxq *nm_rxq;
514 struct sge_nm_txq *nm_txq;
515 int rc, i, j, hwidx;
516 struct hw_buf_info *hwb;
517 uint16_t *rss;
518
519 ASSERT_SYNCHRONIZED_OP(sc);
520
521 if ((vi->flags & VI_INIT_DONE) == 0 ||
522 (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
523 return (EAGAIN);
524
525 hwb = &sc->sge.hw_buf_info[0];
526 for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
527 if (hwb->size == NETMAP_BUF_SIZE(na))
528 break;
529 }
530 if (i >= SGE_FLBUF_SIZES) {
531 if_printf(ifp, "no hwidx for netmap buffer size %d.\n",
532 NETMAP_BUF_SIZE(na));
533 return (ENXIO);
534 }
535 hwidx = i;
536
537 /* Must set caps before calling netmap_reset */
538 nm_set_native_flags(na);
539
540 for_each_nm_rxq(vi, i, nm_rxq) {
541 alloc_nm_rxq_hwq(vi, nm_rxq, tnl_cong(vi->pi, nm_cong_drop));
542 nm_rxq->fl_hwidx = hwidx;
543 slot = netmap_reset(na, NR_RX, i, 0);
544 MPASS(slot != NULL); /* XXXNM: error check, not assert */
545
546 /* We deal with 8 bufs at a time */
547 MPASS((na->num_rx_desc & 7) == 0);
548 MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
549 for (j = 0; j < nm_rxq->fl_sidx; j++) {
550 uint64_t ba;
551
552 PNMB(na, &slot[j], &ba);
553 MPASS(ba != 0);
554 nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
555 }
556 j = nm_rxq->fl_pidx = nm_rxq->fl_sidx - 8;
557 MPASS((j & 7) == 0);
558 j /= 8; /* driver pidx to hardware pidx */
559 wmb();
560 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
561 nm_rxq->fl_db_val | V_PIDX(j));
562 }
563
564 for_each_nm_txq(vi, i, nm_txq) {
565 alloc_nm_txq_hwq(vi, nm_txq);
566 slot = netmap_reset(na, NR_TX, i, 0);
567 MPASS(slot != NULL); /* XXXNM: error check, not assert */
568 }
569
570 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO |
571 M_WAITOK);
572 for (i = 0; i < vi->rss_size;) {
573 for_each_nm_rxq(vi, j, nm_rxq) {
574 rss[i++] = nm_rxq->iq_abs_id;
575 if (i == vi->rss_size)
576 break;
577 }
578 }
579 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size,
580 rss, vi->rss_size);
581 if (rc != 0)
582 if_printf(ifp, "netmap rss_config failed: %d\n", rc);
583 free(rss, M_CXGBE);
584
585 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
586 if (rc != 0)
587 if_printf(ifp, "netmap enable_vi failed: %d\n", rc);
588
589 return (rc);
590 }
591
592 static int
cxgbe_netmap_off(struct adapter * sc,struct vi_info * vi,struct ifnet * ifp,struct netmap_adapter * na)593 cxgbe_netmap_off(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp,
594 struct netmap_adapter *na)
595 {
596 int rc, i;
597 struct sge_nm_txq *nm_txq;
598 struct sge_nm_rxq *nm_rxq;
599
600 ASSERT_SYNCHRONIZED_OP(sc);
601
602 if ((vi->flags & VI_INIT_DONE) == 0)
603 return (0);
604
605 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
606 if (rc != 0)
607 if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
608 nm_clear_native_flags(na);
609
610 for_each_nm_txq(vi, i, nm_txq) {
611 struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];
612
613 /* Wait for hw pidx to catch up ... */
614 while (be16toh(nm_txq->pidx) != spg->pidx)
615 pause("nmpidx", 1);
616
617 /* ... and then for the cidx. */
618 while (spg->pidx != spg->cidx)
619 pause("nmcidx", 1);
620
621 free_nm_txq_hwq(vi, nm_txq);
622 }
623 for_each_nm_rxq(vi, i, nm_rxq) {
624 free_nm_rxq_hwq(vi, nm_rxq);
625 }
626
627 return (rc);
628 }
629
630 static int
cxgbe_netmap_reg(struct netmap_adapter * na,int on)631 cxgbe_netmap_reg(struct netmap_adapter *na, int on)
632 {
633 struct ifnet *ifp = na->ifp;
634 struct vi_info *vi = ifp->if_softc;
635 struct adapter *sc = vi->pi->adapter;
636 int rc;
637
638 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4nmreg");
639 if (rc != 0)
640 return (rc);
641 if (on)
642 rc = cxgbe_netmap_on(sc, vi, ifp, na);
643 else
644 rc = cxgbe_netmap_off(sc, vi, ifp, na);
645 end_synchronized_op(sc, 0);
646
647 return (rc);
648 }
649
650 /* How many packets can a single type1 WR carry in n descriptors */
651 static inline int
ndesc_to_npkt(const int n)652 ndesc_to_npkt(const int n)
653 {
654
655 MPASS(n > 0 && n <= SGE_MAX_WR_NDESC);
656
657 return (n * 2 - 1);
658 }
659 #define MAX_NPKT_IN_TYPE1_WR (ndesc_to_npkt(SGE_MAX_WR_NDESC))
660
661 /* Space (in descriptors) needed for a type1 WR that carries n packets */
662 static inline int
npkt_to_ndesc(const int n)663 npkt_to_ndesc(const int n)
664 {
665
666 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
667
668 return ((n + 2) / 2);
669 }
670
671 /* Space (in 16B units) needed for a type1 WR that carries n packets */
672 static inline int
npkt_to_len16(const int n)673 npkt_to_len16(const int n)
674 {
675
676 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
677
678 return (n * 2 + 1);
679 }
680
681 #define NMIDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->sidx)
682
683 static void
ring_nm_txq_db(struct adapter * sc,struct sge_nm_txq * nm_txq)684 ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)
685 {
686 int n;
687 u_int db = nm_txq->doorbells;
688
689 MPASS(nm_txq->pidx != nm_txq->dbidx);
690
691 n = NMIDXDIFF(nm_txq, dbidx);
692 if (n > 1)
693 clrbit(&db, DOORBELL_WCWR);
694 wmb();
695
696 switch (ffs(db) - 1) {
697 case DOORBELL_UDB:
698 *nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
699 break;
700
701 case DOORBELL_WCWR: {
702 volatile uint64_t *dst, *src;
703
704 /*
705 * Queues whose 128B doorbell segment fits in the page do not
706 * use relative qid (udb_qid is always 0). Only queues with
707 * doorbell segments can do WCWR.
708 */
709 KASSERT(nm_txq->udb_qid == 0 && n == 1,
710 ("%s: inappropriate doorbell (0x%x, %d, %d) for nm_txq %p",
711 __func__, nm_txq->doorbells, n, nm_txq->pidx, nm_txq));
712
713 dst = (volatile void *)((uintptr_t)nm_txq->udb +
714 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
715 src = (void *)&nm_txq->desc[nm_txq->dbidx];
716 while (src != (void *)&nm_txq->desc[nm_txq->dbidx + 1])
717 *dst++ = *src++;
718 wmb();
719 break;
720 }
721
722 case DOORBELL_UDBWC:
723 *nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
724 wmb();
725 break;
726
727 case DOORBELL_KDB:
728 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
729 V_QID(nm_txq->cntxt_id) | V_PIDX(n));
730 break;
731 }
732 nm_txq->dbidx = nm_txq->pidx;
733 }
734
735 int lazy_tx_credit_flush = 1;
736
737 /*
738 * Write work requests to send 'npkt' frames and ring the doorbell to send them
739 * on their way. No need to check for wraparound.
740 */
741 static void
cxgbe_nm_tx(struct adapter * sc,struct sge_nm_txq * nm_txq,struct netmap_kring * kring,int npkt,int npkt_remaining,int txcsum)742 cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,
743 struct netmap_kring *kring, int npkt, int npkt_remaining, int txcsum)
744 {
745 struct netmap_ring *ring = kring->ring;
746 struct netmap_slot *slot;
747 const u_int lim = kring->nkr_num_slots - 1;
748 struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];
749 uint16_t len;
750 uint64_t ba;
751 struct cpl_tx_pkt_core *cpl;
752 struct ulptx_sgl *usgl;
753 int i, n;
754
755 while (npkt) {
756 n = min(npkt, MAX_NPKT_IN_TYPE1_WR);
757 len = 0;
758
759 wr = (void *)&nm_txq->desc[nm_txq->pidx];
760 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
761 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(npkt_to_len16(n)));
762 wr->npkt = n;
763 wr->r3 = 0;
764 wr->type = 1;
765 cpl = (void *)(wr + 1);
766
767 for (i = 0; i < n; i++) {
768 slot = &ring->slot[kring->nr_hwcur];
769 PNMB(kring->na, slot, &ba);
770 MPASS(ba != 0);
771
772 cpl->ctrl0 = nm_txq->cpl_ctrl0;
773 cpl->pack = 0;
774 cpl->len = htobe16(slot->len);
775 /*
776 * netmap(4) says "netmap does not use features such as
777 * checksum offloading, TCP segmentation offloading,
778 * encryption, VLAN encapsulation/decapsulation, etc."
779 *
780 * So the ncxl interfaces have tx hardware checksumming
781 * disabled by default. But you can override netmap by
782 * enabling IFCAP_TXCSUM on the interface manully.
783 */
784 cpl->ctrl1 = txcsum ? 0 :
785 htobe64(F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS);
786
787 usgl = (void *)(cpl + 1);
788 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
789 V_ULPTX_NSGE(1));
790 usgl->len0 = htobe32(slot->len);
791 usgl->addr0 = htobe64(ba);
792
793 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
794 cpl = (void *)(usgl + 1);
795 MPASS(slot->len + len <= UINT16_MAX);
796 len += slot->len;
797 kring->nr_hwcur = nm_next(kring->nr_hwcur, lim);
798 }
799 wr->plen = htobe16(len);
800
801 npkt -= n;
802 nm_txq->pidx += npkt_to_ndesc(n);
803 MPASS(nm_txq->pidx <= nm_txq->sidx);
804 if (__predict_false(nm_txq->pidx == nm_txq->sidx)) {
805 /*
806 * This routine doesn't know how to write WRs that wrap
807 * around. Make sure it wasn't asked to.
808 */
809 MPASS(npkt == 0);
810 nm_txq->pidx = 0;
811 }
812
813 if (npkt == 0 && npkt_remaining == 0) {
814 /* All done. */
815 if (lazy_tx_credit_flush == 0) {
816 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
817 F_FW_WR_EQUIQ);
818 nm_txq->equeqidx = nm_txq->pidx;
819 nm_txq->equiqidx = nm_txq->pidx;
820 }
821 ring_nm_txq_db(sc, nm_txq);
822 return;
823 }
824
825 if (NMIDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {
826 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
827 F_FW_WR_EQUIQ);
828 nm_txq->equeqidx = nm_txq->pidx;
829 nm_txq->equiqidx = nm_txq->pidx;
830 } else if (NMIDXDIFF(nm_txq, equeqidx) >= 64) {
831 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
832 nm_txq->equeqidx = nm_txq->pidx;
833 }
834 if (NMIDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC)
835 ring_nm_txq_db(sc, nm_txq);
836 }
837
838 /* Will get called again. */
839 MPASS(npkt_remaining);
840 }
841
842 /* How many contiguous free descriptors starting at pidx */
843 static inline int
contiguous_ndesc_available(struct sge_nm_txq * nm_txq)844 contiguous_ndesc_available(struct sge_nm_txq *nm_txq)
845 {
846
847 if (nm_txq->cidx > nm_txq->pidx)
848 return (nm_txq->cidx - nm_txq->pidx - 1);
849 else if (nm_txq->cidx > 0)
850 return (nm_txq->sidx - nm_txq->pidx);
851 else
852 return (nm_txq->sidx - nm_txq->pidx - 1);
853 }
854
855 static int
reclaim_nm_tx_desc(struct sge_nm_txq * nm_txq)856 reclaim_nm_tx_desc(struct sge_nm_txq *nm_txq)
857 {
858 struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];
859 uint16_t hw_cidx = spg->cidx; /* snapshot */
860 struct fw_eth_tx_pkts_wr *wr;
861 int n = 0;
862
863 hw_cidx = be16toh(hw_cidx);
864
865 while (nm_txq->cidx != hw_cidx) {
866 wr = (void *)&nm_txq->desc[nm_txq->cidx];
867
868 MPASS(wr->op_pkd == htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)));
869 MPASS(wr->type == 1);
870 MPASS(wr->npkt > 0 && wr->npkt <= MAX_NPKT_IN_TYPE1_WR);
871
872 n += wr->npkt;
873 nm_txq->cidx += npkt_to_ndesc(wr->npkt);
874
875 /*
876 * We never sent a WR that wrapped around so the credits coming
877 * back, WR by WR, should never cause the cidx to wrap around
878 * either.
879 */
880 MPASS(nm_txq->cidx <= nm_txq->sidx);
881 if (__predict_false(nm_txq->cidx == nm_txq->sidx))
882 nm_txq->cidx = 0;
883 }
884
885 return (n);
886 }
887
888 static int
cxgbe_netmap_txsync(struct netmap_kring * kring,int flags)889 cxgbe_netmap_txsync(struct netmap_kring *kring, int flags)
890 {
891 struct netmap_adapter *na = kring->na;
892 struct ifnet *ifp = na->ifp;
893 struct vi_info *vi = ifp->if_softc;
894 struct adapter *sc = vi->pi->adapter;
895 struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[vi->first_txq + kring->ring_id];
896 const u_int head = kring->rhead;
897 u_int reclaimed = 0;
898 int n, d, npkt_remaining, ndesc_remaining, txcsum;
899
900 /*
901 * Tx was at kring->nr_hwcur last time around and now we need to advance
902 * to kring->rhead. Note that the driver's pidx moves independent of
903 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation
904 * between descriptors and frames isn't 1:1).
905 */
906
907 npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
908 kring->nkr_num_slots - kring->nr_hwcur + head;
909 txcsum = ifp->if_capenable & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6);
910 while (npkt_remaining) {
911 reclaimed += reclaim_nm_tx_desc(nm_txq);
912 ndesc_remaining = contiguous_ndesc_available(nm_txq);
913 /* Can't run out of descriptors with packets still remaining */
914 MPASS(ndesc_remaining > 0);
915
916 /* # of desc needed to tx all remaining packets */
917 d = (npkt_remaining / MAX_NPKT_IN_TYPE1_WR) * SGE_MAX_WR_NDESC;
918 if (npkt_remaining % MAX_NPKT_IN_TYPE1_WR)
919 d += npkt_to_ndesc(npkt_remaining % MAX_NPKT_IN_TYPE1_WR);
920
921 if (d <= ndesc_remaining)
922 n = npkt_remaining;
923 else {
924 /* Can't send all, calculate how many can be sent */
925 n = (ndesc_remaining / SGE_MAX_WR_NDESC) *
926 MAX_NPKT_IN_TYPE1_WR;
927 if (ndesc_remaining % SGE_MAX_WR_NDESC)
928 n += ndesc_to_npkt(ndesc_remaining % SGE_MAX_WR_NDESC);
929 }
930
931 /* Send n packets and update nm_txq->pidx and kring->nr_hwcur */
932 npkt_remaining -= n;
933 cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining, txcsum);
934 }
935 MPASS(npkt_remaining == 0);
936 MPASS(kring->nr_hwcur == head);
937 MPASS(nm_txq->dbidx == nm_txq->pidx);
938
939 /*
940 * Second part: reclaim buffers for completed transmissions.
941 */
942 if (reclaimed || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {
943 reclaimed += reclaim_nm_tx_desc(nm_txq);
944 kring->nr_hwtail += reclaimed;
945 if (kring->nr_hwtail >= kring->nkr_num_slots)
946 kring->nr_hwtail -= kring->nkr_num_slots;
947 }
948
949 return (0);
950 }
951
952 static int
cxgbe_netmap_rxsync(struct netmap_kring * kring,int flags)953 cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags)
954 {
955 struct netmap_adapter *na = kring->na;
956 struct netmap_ring *ring = kring->ring;
957 struct ifnet *ifp = na->ifp;
958 struct vi_info *vi = ifp->if_softc;
959 struct adapter *sc = vi->pi->adapter;
960 struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[vi->first_rxq + kring->ring_id];
961 u_int const head = kring->rhead;
962 u_int n;
963 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
964
965 if (black_hole)
966 return (0); /* No updates ever. */
967
968 if (netmap_no_pendintr || force_update) {
969 kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
970 kring->nr_kflags &= ~NKR_PENDINTR;
971 }
972
973 /* Userspace done with buffers from kring->nr_hwcur to head */
974 n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
975 kring->nkr_num_slots - kring->nr_hwcur + head;
976 n &= ~7U;
977 if (n > 0) {
978 u_int fl_pidx = nm_rxq->fl_pidx;
979 struct netmap_slot *slot = &ring->slot[fl_pidx];
980 uint64_t ba;
981 int i, dbinc = 0, hwidx = nm_rxq->fl_hwidx;
982
983 /*
984 * We always deal with 8 buffers at a time. We must have
985 * stopped at an 8B boundary (fl_pidx) last time around and we
986 * must have a multiple of 8B buffers to give to the freelist.
987 */
988 MPASS((fl_pidx & 7) == 0);
989 MPASS((n & 7) == 0);
990
991 IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
992 IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx);
993
994 while (n > 0) {
995 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
996 PNMB(na, slot, &ba);
997 MPASS(ba != 0);
998 nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
999 slot->flags &= ~NS_BUF_CHANGED;
1000 MPASS(fl_pidx <= nm_rxq->fl_sidx);
1001 }
1002 n -= 8;
1003 if (fl_pidx == nm_rxq->fl_sidx) {
1004 fl_pidx = 0;
1005 slot = &ring->slot[0];
1006 }
1007 if (++dbinc == 8 && n >= 32) {
1008 wmb();
1009 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1010 nm_rxq->fl_db_val | V_PIDX(dbinc));
1011 dbinc = 0;
1012 }
1013 }
1014 MPASS(nm_rxq->fl_pidx == fl_pidx);
1015
1016 if (dbinc > 0) {
1017 wmb();
1018 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1019 nm_rxq->fl_db_val | V_PIDX(dbinc));
1020 }
1021 }
1022
1023 return (0);
1024 }
1025
1026 static int
ncxgbe_probe(device_t dev)1027 ncxgbe_probe(device_t dev)
1028 {
1029 char buf[128];
1030 struct vi_info *vi = device_get_softc(dev);
1031
1032 snprintf(buf, sizeof(buf), "port %d netmap vi", vi->pi->port_id);
1033 device_set_desc_copy(dev, buf);
1034
1035 return (BUS_PROBE_DEFAULT);
1036 }
1037
1038 static int
ncxgbe_attach(device_t dev)1039 ncxgbe_attach(device_t dev)
1040 {
1041 struct vi_info *vi;
1042 struct port_info *pi;
1043 struct adapter *sc;
1044 struct netmap_adapter na;
1045 struct ifnet *ifp;
1046 int rc;
1047
1048 vi = device_get_softc(dev);
1049 pi = vi->pi;
1050 sc = pi->adapter;
1051
1052 /*
1053 * Allocate a virtual interface exclusively for netmap use. Give it the
1054 * MAC address normally reserved for use by a TOE interface. (The TOE
1055 * driver on FreeBSD doesn't use it).
1056 */
1057 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
1058 vi->hw_addr, &vi->rss_size, FW_VI_FUNC_OFLD, 0);
1059 if (rc < 0) {
1060 device_printf(dev, "unable to allocate netmap virtual "
1061 "interface for port %d: %d\n", pi->port_id, -rc);
1062 return (-rc);
1063 }
1064 vi->viid = rc;
1065 vi->xact_addr_filt = -1;
1066 callout_init(&vi->tick, 1);
1067
1068 ifp = if_alloc(IFT_ETHER);
1069 if (ifp == NULL) {
1070 device_printf(dev, "Cannot allocate netmap ifnet\n");
1071 return (ENOMEM);
1072 }
1073 vi->ifp = ifp;
1074 ifp->if_softc = vi;
1075
1076 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1077 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1078
1079 ifp->if_init = cxgbe_nm_init;
1080 ifp->if_ioctl = cxgbe_nm_ioctl;
1081 ifp->if_transmit = cxgbe_nm_transmit;
1082 ifp->if_qflush = cxgbe_nm_qflush;
1083 ifp->if_get_counter = cxgbe_get_counter;
1084
1085 /*
1086 * netmap(4) says "netmap does not use features such as checksum
1087 * offloading, TCP segmentation offloading, encryption, VLAN
1088 * encapsulation/decapsulation, etc."
1089 *
1090 * By default we comply with the statement above. But we do declare the
1091 * ifnet capable of L3/L4 checksumming so that a user can override
1092 * netmap and have the hardware do the L3/L4 checksums.
1093 */
1094 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU |
1095 IFCAP_HWCSUM_IPV6;
1096 ifp->if_capenable = 0;
1097 ifp->if_hwassist = 0;
1098
1099 /* vi->media has already been setup by the caller */
1100
1101 ether_ifattach(ifp, vi->hw_addr);
1102
1103 device_printf(dev, "%d txq, %d rxq (netmap)\n", vi->ntxq, vi->nrxq);
1104
1105 vi_sysctls(vi);
1106
1107 /*
1108 * Register with netmap in the kernel.
1109 */
1110 bzero(&na, sizeof(na));
1111
1112 na.ifp = ifp;
1113 na.na_flags = NAF_BDG_MAYSLEEP;
1114
1115 /* Netmap doesn't know about the space reserved for the status page. */
1116 na.num_tx_desc = vi->qsize_txq - spg_len / EQ_ESIZE;
1117
1118 /*
1119 * The freelist's cidx/pidx drives netmap's rx cidx/pidx. So
1120 * num_rx_desc is based on the number of buffers that can be held in the
1121 * freelist, and not the number of entries in the iq. (These two are
1122 * not exactly the same due to the space taken up by the status page).
1123 */
1124 na.num_rx_desc = (vi->qsize_rxq / 8) * 8;
1125 na.nm_txsync = cxgbe_netmap_txsync;
1126 na.nm_rxsync = cxgbe_netmap_rxsync;
1127 na.nm_register = cxgbe_netmap_reg;
1128 na.num_tx_rings = vi->ntxq;
1129 na.num_rx_rings = vi->nrxq;
1130 netmap_attach(&na); /* This adds IFCAP_NETMAP to if_capabilities */
1131
1132 return (0);
1133 }
1134
1135 static int
ncxgbe_detach(device_t dev)1136 ncxgbe_detach(device_t dev)
1137 {
1138 struct vi_info *vi;
1139 struct adapter *sc;
1140
1141 vi = device_get_softc(dev);
1142 sc = vi->pi->adapter;
1143
1144 doom_vi(sc, vi);
1145
1146 netmap_detach(vi->ifp);
1147 ether_ifdetach(vi->ifp);
1148 cxgbe_nm_uninit_synchronized(vi);
1149 callout_drain(&vi->tick);
1150 vi_full_uninit(vi);
1151 ifmedia_removeall(&vi->media);
1152 if_free(vi->ifp);
1153 vi->ifp = NULL;
1154 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1155
1156 end_synchronized_op(sc, 0);
1157
1158 return (0);
1159 }
1160
1161 static void
handle_nm_fw6_msg(struct adapter * sc,struct ifnet * ifp,const struct cpl_fw6_msg * cpl)1162 handle_nm_fw6_msg(struct adapter *sc, struct ifnet *ifp,
1163 const struct cpl_fw6_msg *cpl)
1164 {
1165 const struct cpl_sge_egr_update *egr;
1166 uint32_t oq;
1167 struct sge_nm_txq *nm_txq;
1168
1169 if (cpl->type != FW_TYPE_RSSCPL && cpl->type != FW6_TYPE_RSSCPL)
1170 panic("%s: FW_TYPE 0x%x on nm_rxq.", __func__, cpl->type);
1171
1172 /* data[0] is RSS header */
1173 egr = (const void *)&cpl->data[1];
1174 oq = be32toh(egr->opcode_qid);
1175 MPASS(G_CPL_OPCODE(oq) == CPL_SGE_EGR_UPDATE);
1176 nm_txq = (void *)sc->sge.eqmap[G_EGR_QID(oq) - sc->sge.eq_start];
1177
1178 netmap_tx_irq(ifp, nm_txq->nid);
1179 }
1180
1181 void
t4_nm_intr(void * arg)1182 t4_nm_intr(void *arg)
1183 {
1184 struct sge_nm_rxq *nm_rxq = arg;
1185 struct vi_info *vi = nm_rxq->vi;
1186 struct adapter *sc = vi->pi->adapter;
1187 struct ifnet *ifp = vi->ifp;
1188 struct netmap_adapter *na = NA(ifp);
1189 struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
1190 struct netmap_ring *ring = kring->ring;
1191 struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
1192 uint32_t lq;
1193 u_int n = 0, work = 0;
1194 uint8_t opcode;
1195 uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
1196 u_int fl_credits = fl_cidx & 7;
1197
1198 while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
1199
1200 rmb();
1201
1202 lq = be32toh(d->rsp.pldbuflen_qid);
1203 opcode = d->rss.opcode;
1204
1205 switch (G_RSPD_TYPE(d->rsp.u.type_gen)) {
1206 case X_RSPD_TYPE_FLBUF:
1207 if (black_hole != 2) {
1208 /* No buffer packing so new buf every time */
1209 MPASS(lq & F_RSPD_NEWBUF);
1210 }
1211
1212 /* fall through */
1213
1214 case X_RSPD_TYPE_CPL:
1215 MPASS(opcode < NUM_CPL_CMDS);
1216
1217 switch (opcode) {
1218 case CPL_FW4_MSG:
1219 case CPL_FW6_MSG:
1220 handle_nm_fw6_msg(sc, ifp,
1221 (const void *)&d->cpl[0]);
1222 break;
1223 case CPL_RX_PKT:
1224 ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
1225 ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
1226 fl_cidx += (lq & F_RSPD_NEWBUF) ? 1 : 0;
1227 fl_credits += (lq & F_RSPD_NEWBUF) ? 1 : 0;
1228 if (__predict_false(fl_cidx == nm_rxq->fl_sidx))
1229 fl_cidx = 0;
1230 break;
1231 default:
1232 panic("%s: unexpected opcode 0x%x on nm_rxq %p",
1233 __func__, opcode, nm_rxq);
1234 }
1235 break;
1236
1237 case X_RSPD_TYPE_INTR:
1238 /* Not equipped to handle forwarded interrupts. */
1239 panic("%s: netmap queue received interrupt for iq %u\n",
1240 __func__, lq);
1241
1242 default:
1243 panic("%s: illegal response type %d on nm_rxq %p",
1244 __func__, G_RSPD_TYPE(d->rsp.u.type_gen), nm_rxq);
1245 }
1246
1247 d++;
1248 if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {
1249 nm_rxq->iq_cidx = 0;
1250 d = &nm_rxq->iq_desc[0];
1251 nm_rxq->iq_gen ^= F_RSPD_GEN;
1252 }
1253
1254 if (__predict_false(++n == rx_ndesc)) {
1255 atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
1256 if (black_hole && fl_credits >= 8) {
1257 fl_credits /= 8;
1258 IDXINCR(nm_rxq->fl_pidx, fl_credits * 8,
1259 nm_rxq->fl_sidx);
1260 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1261 nm_rxq->fl_db_val | V_PIDX(fl_credits));
1262 fl_credits = fl_cidx & 7;
1263 } else if (!black_hole) {
1264 netmap_rx_irq(ifp, nm_rxq->nid, &work);
1265 MPASS(work != 0);
1266 }
1267 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1268 V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
1269 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1270 n = 0;
1271 }
1272 }
1273
1274 atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
1275 if (black_hole) {
1276 fl_credits /= 8;
1277 IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx);
1278 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1279 nm_rxq->fl_db_val | V_PIDX(fl_credits));
1280 } else
1281 netmap_rx_irq(ifp, nm_rxq->nid, &work);
1282
1283 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
1284 V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) |
1285 V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
1286 }
1287
1288 static devclass_t ncxgbe_devclass, ncxl_devclass;
1289
1290 DRIVER_MODULE(ncxgbe, cxgbe, ncxgbe_driver, ncxgbe_devclass, 0, 0);
1291 MODULE_VERSION(ncxgbe, 1);
1292
1293 DRIVER_MODULE(ncxl, cxl, ncxl_driver, ncxl_devclass, 0, 0);
1294 MODULE_VERSION(ncxl, 1);
1295 #endif
1296