1 /*-
2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3 * All rights reserved.
4 *
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/types.h>
34 #include <sys/mbuf.h>
35 #include <sys/smp.h>
36 #include <sys/socket.h>
37 #include <sys/sysctl.h>
38 #include <sys/limits.h>
39 #include <sys/syslog.h>
40
41 #include <net/ethernet.h>
42 #include <net/if.h>
43 #include <net/if_vlan_var.h>
44
45 #include <netinet/in.h>
46 #include <netinet/ip.h>
47 #include <netinet/ip6.h>
48 #include <netinet/tcp.h>
49
50 #include <machine/in_cksum.h>
51
52 #include "common/efx.h"
53
54
55 #include "sfxge.h"
56 #include "sfxge_rx.h"
57
58 #define RX_REFILL_THRESHOLD(_entries) (EFX_RXQ_LIMIT(_entries) * 9 / 10)
59
60 #ifdef SFXGE_LRO
61
62 SYSCTL_NODE(_hw_sfxge, OID_AUTO, lro, CTLFLAG_RD, NULL,
63 "Large receive offload (LRO) parameters");
64
65 #define SFXGE_LRO_PARAM(_param) SFXGE_PARAM(lro._param)
66
67 /* Size of the LRO hash table. Must be a power of 2. A larger table
68 * means we can accelerate a larger number of streams.
69 */
70 static unsigned lro_table_size = 128;
71 TUNABLE_INT(SFXGE_LRO_PARAM(table_size), &lro_table_size);
72 SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, table_size, CTLFLAG_RDTUN,
73 &lro_table_size, 0,
74 "Size of the LRO hash table (must be a power of 2)");
75
76 /* Maximum length of a hash chain. If chains get too long then the lookup
77 * time increases and may exceed the benefit of LRO.
78 */
79 static unsigned lro_chain_max = 20;
80 TUNABLE_INT(SFXGE_LRO_PARAM(chain_max), &lro_chain_max);
81 SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, chain_max, CTLFLAG_RDTUN,
82 &lro_chain_max, 0,
83 "The maximum length of a hash chain");
84
85 /* Maximum time (in ticks) that a connection can be idle before it's LRO
86 * state is discarded.
87 */
88 static unsigned lro_idle_ticks; /* initialised in sfxge_rx_init() */
89 TUNABLE_INT(SFXGE_LRO_PARAM(idle_ticks), &lro_idle_ticks);
90 SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, idle_ticks, CTLFLAG_RDTUN,
91 &lro_idle_ticks, 0,
92 "The maximum time (in ticks) that a connection can be idle "
93 "before it's LRO state is discarded");
94
95 /* Number of packets with payload that must arrive in-order before a
96 * connection is eligible for LRO. The idea is we should avoid coalescing
97 * segments when the sender is in slow-start because reducing the ACK rate
98 * can damage performance.
99 */
100 static int lro_slow_start_packets = 2000;
101 TUNABLE_INT(SFXGE_LRO_PARAM(slow_start_packets), &lro_slow_start_packets);
102 SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, slow_start_packets, CTLFLAG_RDTUN,
103 &lro_slow_start_packets, 0,
104 "Number of packets with payload that must arrive in-order before "
105 "a connection is eligible for LRO");
106
107 /* Number of packets with payload that must arrive in-order following loss
108 * before a connection is eligible for LRO. The idea is we should avoid
109 * coalescing segments when the sender is recovering from loss, because
110 * reducing the ACK rate can damage performance.
111 */
112 static int lro_loss_packets = 20;
113 TUNABLE_INT(SFXGE_LRO_PARAM(loss_packets), &lro_loss_packets);
114 SYSCTL_UINT(_hw_sfxge_lro, OID_AUTO, loss_packets, CTLFLAG_RDTUN,
115 &lro_loss_packets, 0,
116 "Number of packets with payload that must arrive in-order "
117 "following loss before a connection is eligible for LRO");
118
119 /* Flags for sfxge_lro_conn::l2_id; must not collide with EVL_VLID_MASK */
120 #define SFXGE_LRO_L2_ID_VLAN 0x4000
121 #define SFXGE_LRO_L2_ID_IPV6 0x8000
122 #define SFXGE_LRO_CONN_IS_VLAN_ENCAP(c) ((c)->l2_id & SFXGE_LRO_L2_ID_VLAN)
123 #define SFXGE_LRO_CONN_IS_TCPIPV4(c) (!((c)->l2_id & SFXGE_LRO_L2_ID_IPV6))
124
125 /* Compare IPv6 addresses, avoiding conditional branches */
ipv6_addr_cmp(const struct in6_addr * left,const struct in6_addr * right)126 static unsigned long ipv6_addr_cmp(const struct in6_addr *left,
127 const struct in6_addr *right)
128 {
129 #if LONG_BIT == 64
130 const uint64_t *left64 = (const uint64_t *)left;
131 const uint64_t *right64 = (const uint64_t *)right;
132 return (left64[0] - right64[0]) | (left64[1] - right64[1]);
133 #else
134 return (left->s6_addr32[0] - right->s6_addr32[0]) |
135 (left->s6_addr32[1] - right->s6_addr32[1]) |
136 (left->s6_addr32[2] - right->s6_addr32[2]) |
137 (left->s6_addr32[3] - right->s6_addr32[3]);
138 #endif
139 }
140
141 #endif /* SFXGE_LRO */
142
143 void
sfxge_rx_qflush_done(struct sfxge_rxq * rxq)144 sfxge_rx_qflush_done(struct sfxge_rxq *rxq)
145 {
146
147 rxq->flush_state = SFXGE_FLUSH_DONE;
148 }
149
150 void
sfxge_rx_qflush_failed(struct sfxge_rxq * rxq)151 sfxge_rx_qflush_failed(struct sfxge_rxq *rxq)
152 {
153
154 rxq->flush_state = SFXGE_FLUSH_FAILED;
155 }
156
157 static uint8_t toep_key[] = {
158 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
159 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
160 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
161 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
162 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa
163 };
164
165 static void
sfxge_rx_post_refill(void * arg)166 sfxge_rx_post_refill(void *arg)
167 {
168 struct sfxge_rxq *rxq = arg;
169 struct sfxge_softc *sc;
170 unsigned int index;
171 struct sfxge_evq *evq;
172 uint16_t magic;
173
174 sc = rxq->sc;
175 index = rxq->index;
176 evq = sc->evq[index];
177
178 magic = SFXGE_MAGIC_RX_QREFILL | index;
179
180 /* This is guaranteed due to the start/stop order of rx and ev */
181 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
182 ("evq not started"));
183 KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
184 ("rxq not started"));
185 efx_ev_qpost(evq->common, magic);
186 }
187
188 static void
sfxge_rx_schedule_refill(struct sfxge_rxq * rxq,boolean_t retrying)189 sfxge_rx_schedule_refill(struct sfxge_rxq *rxq, boolean_t retrying)
190 {
191 /* Initially retry after 100 ms, but back off in case of
192 * repeated failures as we probably have to wait for the
193 * administrator to raise the pool limit. */
194 if (retrying)
195 rxq->refill_delay = min(rxq->refill_delay * 2, 10 * hz);
196 else
197 rxq->refill_delay = hz / 10;
198
199 callout_reset_curcpu(&rxq->refill_callout, rxq->refill_delay,
200 sfxge_rx_post_refill, rxq);
201 }
202
sfxge_rx_alloc_mbuf(struct sfxge_softc * sc)203 static struct mbuf *sfxge_rx_alloc_mbuf(struct sfxge_softc *sc)
204 {
205 struct mb_args args;
206 struct mbuf *m;
207
208 /* Allocate mbuf structure */
209 args.flags = M_PKTHDR;
210 args.type = MT_DATA;
211 m = (struct mbuf *)uma_zalloc_arg(zone_mbuf, &args, M_NOWAIT);
212
213 /* Allocate (and attach) packet buffer */
214 if (m != NULL && !uma_zalloc_arg(sc->rx_buffer_zone, m, M_NOWAIT)) {
215 uma_zfree(zone_mbuf, m);
216 m = NULL;
217 }
218
219 return (m);
220 }
221
222 #define SFXGE_REFILL_BATCH 64
223
224 static void
sfxge_rx_qfill(struct sfxge_rxq * rxq,unsigned int target,boolean_t retrying)225 sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying)
226 {
227 struct sfxge_softc *sc;
228 unsigned int index;
229 struct sfxge_evq *evq;
230 unsigned int batch;
231 unsigned int rxfill;
232 unsigned int mblksize;
233 int ntodo;
234 efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
235
236 sc = rxq->sc;
237 index = rxq->index;
238 evq = sc->evq[index];
239
240 prefetch_read_many(sc->enp);
241 prefetch_read_many(rxq->common);
242
243 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
244
245 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
246 return;
247
248 rxfill = rxq->added - rxq->completed;
249 KASSERT(rxfill <= EFX_RXQ_LIMIT(rxq->entries),
250 ("rxfill > EFX_RXQ_LIMIT(rxq->entries)"));
251 ntodo = min(EFX_RXQ_LIMIT(rxq->entries) - rxfill, target);
252 KASSERT(ntodo <= EFX_RXQ_LIMIT(rxq->entries),
253 ("ntodo > EFX_RQX_LIMIT(rxq->entries)"));
254
255 if (ntodo == 0)
256 return;
257
258 batch = 0;
259 mblksize = sc->rx_buffer_size;
260 while (ntodo-- > 0) {
261 unsigned int id;
262 struct sfxge_rx_sw_desc *rx_desc;
263 bus_dma_segment_t seg;
264 struct mbuf *m;
265
266 id = (rxq->added + batch) & rxq->ptr_mask;
267 rx_desc = &rxq->queue[id];
268 KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
269
270 rx_desc->flags = EFX_DISCARD;
271 m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
272 if (m == NULL)
273 break;
274 sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);
275 addr[batch++] = seg.ds_addr;
276
277 if (batch == SFXGE_REFILL_BATCH) {
278 efx_rx_qpost(rxq->common, addr, mblksize, batch,
279 rxq->completed, rxq->added);
280 rxq->added += batch;
281 batch = 0;
282 }
283 }
284
285 if (ntodo != 0)
286 sfxge_rx_schedule_refill(rxq, retrying);
287
288 if (batch != 0) {
289 efx_rx_qpost(rxq->common, addr, mblksize, batch,
290 rxq->completed, rxq->added);
291 rxq->added += batch;
292 }
293
294 /* Make the descriptors visible to the hardware */
295 bus_dmamap_sync(rxq->mem.esm_tag, rxq->mem.esm_map,
296 BUS_DMASYNC_PREWRITE);
297
298 efx_rx_qpush(rxq->common, rxq->added);
299 }
300
301 void
sfxge_rx_qrefill(struct sfxge_rxq * rxq)302 sfxge_rx_qrefill(struct sfxge_rxq *rxq)
303 {
304
305 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
306 return;
307
308 /* Make sure the queue is full */
309 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_TRUE);
310 }
311
__sfxge_rx_deliver(struct sfxge_softc * sc,struct mbuf * m)312 static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m)
313 {
314 struct ifnet *ifp = sc->ifnet;
315
316 m->m_pkthdr.rcvif = ifp;
317 m->m_pkthdr.csum_data = 0xffff;
318 ifp->if_input(ifp, m);
319 }
320
321 static void
sfxge_rx_deliver(struct sfxge_softc * sc,struct sfxge_rx_sw_desc * rx_desc)322 sfxge_rx_deliver(struct sfxge_softc *sc, struct sfxge_rx_sw_desc *rx_desc)
323 {
324 struct mbuf *m = rx_desc->mbuf;
325 int flags = rx_desc->flags;
326 int csum_flags;
327
328 /* Convert checksum flags */
329 csum_flags = (flags & EFX_CKSUM_IPV4) ?
330 (CSUM_IP_CHECKED | CSUM_IP_VALID) : 0;
331 if (flags & EFX_CKSUM_TCPUDP)
332 csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
333
334 /* The hash covers a 4-tuple for TCP only */
335 if (flags & EFX_PKT_TCP) {
336 m->m_pkthdr.flowid = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
337 mtod(m, uint8_t *));
338 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
339 }
340 m->m_data += sc->rx_prefix_size;
341 m->m_len = rx_desc->size - sc->rx_prefix_size;
342 m->m_pkthdr.len = m->m_len;
343 m->m_pkthdr.csum_flags = csum_flags;
344 __sfxge_rx_deliver(sc, rx_desc->mbuf);
345
346 rx_desc->flags = EFX_DISCARD;
347 rx_desc->mbuf = NULL;
348 }
349
350 #ifdef SFXGE_LRO
351
352 static void
sfxge_lro_deliver(struct sfxge_lro_state * st,struct sfxge_lro_conn * c)353 sfxge_lro_deliver(struct sfxge_lro_state *st, struct sfxge_lro_conn *c)
354 {
355 struct sfxge_softc *sc = st->sc;
356 struct mbuf *m = c->mbuf;
357 struct tcphdr *c_th;
358 int csum_flags;
359
360 KASSERT(m, ("no mbuf to deliver"));
361
362 ++st->n_bursts;
363
364 /* Finish off packet munging and recalculate IP header checksum. */
365 if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
366 struct ip *iph = c->nh;
367 iph->ip_len = htons(iph->ip_len);
368 iph->ip_sum = 0;
369 iph->ip_sum = in_cksum_hdr(iph);
370 c_th = (struct tcphdr *)(iph + 1);
371 csum_flags = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
372 CSUM_IP_CHECKED | CSUM_IP_VALID);
373 } else {
374 struct ip6_hdr *iph = c->nh;
375 iph->ip6_plen = htons(iph->ip6_plen);
376 c_th = (struct tcphdr *)(iph + 1);
377 csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
378 }
379
380 c_th->th_win = c->th_last->th_win;
381 c_th->th_ack = c->th_last->th_ack;
382 if (c_th->th_off == c->th_last->th_off) {
383 /* Copy TCP options (take care to avoid going negative). */
384 int optlen = ((c_th->th_off - 5) & 0xf) << 2u;
385 memcpy(c_th + 1, c->th_last + 1, optlen);
386 }
387
388 m->m_pkthdr.flowid = c->conn_hash;
389 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
390
391 m->m_pkthdr.csum_flags = csum_flags;
392 __sfxge_rx_deliver(sc, m);
393
394 c->mbuf = NULL;
395 c->delivered = 1;
396 }
397
398 /* Drop the given connection, and add it to the free list. */
sfxge_lro_drop(struct sfxge_rxq * rxq,struct sfxge_lro_conn * c)399 static void sfxge_lro_drop(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
400 {
401 unsigned bucket;
402
403 KASSERT(!c->mbuf, ("found orphaned mbuf"));
404
405 if (c->next_buf.mbuf != NULL) {
406 sfxge_rx_deliver(rxq->sc, &c->next_buf);
407 LIST_REMOVE(c, active_link);
408 }
409
410 bucket = c->conn_hash & rxq->lro.conns_mask;
411 KASSERT(rxq->lro.conns_n[bucket] > 0, ("LRO: bucket fill level wrong"));
412 --rxq->lro.conns_n[bucket];
413 TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
414 TAILQ_INSERT_HEAD(&rxq->lro.free_conns, c, link);
415 }
416
417 /* Stop tracking connections that have gone idle in order to keep hash
418 * chains short.
419 */
sfxge_lro_purge_idle(struct sfxge_rxq * rxq,unsigned now)420 static void sfxge_lro_purge_idle(struct sfxge_rxq *rxq, unsigned now)
421 {
422 struct sfxge_lro_conn *c;
423 unsigned i;
424
425 KASSERT(LIST_EMPTY(&rxq->lro.active_conns),
426 ("found active connections"));
427
428 rxq->lro.last_purge_ticks = now;
429 for (i = 0; i <= rxq->lro.conns_mask; ++i) {
430 if (TAILQ_EMPTY(&rxq->lro.conns[i]))
431 continue;
432
433 c = TAILQ_LAST(&rxq->lro.conns[i], sfxge_lro_tailq);
434 if (now - c->last_pkt_ticks > lro_idle_ticks) {
435 ++rxq->lro.n_drop_idle;
436 sfxge_lro_drop(rxq, c);
437 }
438 }
439 }
440
441 static void
sfxge_lro_merge(struct sfxge_lro_state * st,struct sfxge_lro_conn * c,struct mbuf * mbuf,struct tcphdr * th)442 sfxge_lro_merge(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
443 struct mbuf *mbuf, struct tcphdr *th)
444 {
445 struct tcphdr *c_th;
446
447 /* Tack the new mbuf onto the chain. */
448 KASSERT(!mbuf->m_next, ("mbuf already chained"));
449 c->mbuf_tail->m_next = mbuf;
450 c->mbuf_tail = mbuf;
451
452 /* Increase length appropriately */
453 c->mbuf->m_pkthdr.len += mbuf->m_len;
454
455 /* Update the connection state flags */
456 if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
457 struct ip *iph = c->nh;
458 iph->ip_len += mbuf->m_len;
459 c_th = (struct tcphdr *)(iph + 1);
460 } else {
461 struct ip6_hdr *iph = c->nh;
462 iph->ip6_plen += mbuf->m_len;
463 c_th = (struct tcphdr *)(iph + 1);
464 }
465 c_th->th_flags |= (th->th_flags & TH_PUSH);
466 c->th_last = th;
467 ++st->n_merges;
468
469 /* Pass packet up now if another segment could overflow the IP
470 * length.
471 */
472 if (c->mbuf->m_pkthdr.len > 65536 - 9200)
473 sfxge_lro_deliver(st, c);
474 }
475
476 static void
sfxge_lro_start(struct sfxge_lro_state * st,struct sfxge_lro_conn * c,struct mbuf * mbuf,void * nh,struct tcphdr * th)477 sfxge_lro_start(struct sfxge_lro_state *st, struct sfxge_lro_conn *c,
478 struct mbuf *mbuf, void *nh, struct tcphdr *th)
479 {
480 /* Start the chain */
481 c->mbuf = mbuf;
482 c->mbuf_tail = c->mbuf;
483 c->nh = nh;
484 c->th_last = th;
485
486 mbuf->m_pkthdr.len = mbuf->m_len;
487
488 /* Mangle header fields for later processing */
489 if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
490 struct ip *iph = nh;
491 iph->ip_len = ntohs(iph->ip_len);
492 } else {
493 struct ip6_hdr *iph = nh;
494 iph->ip6_plen = ntohs(iph->ip6_plen);
495 }
496 }
497
498 /* Try to merge or otherwise hold or deliver (as appropriate) the
499 * packet buffered for this connection (c->next_buf). Return a flag
500 * indicating whether the connection is still active for LRO purposes.
501 */
502 static int
sfxge_lro_try_merge(struct sfxge_rxq * rxq,struct sfxge_lro_conn * c)503 sfxge_lro_try_merge(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c)
504 {
505 struct sfxge_rx_sw_desc *rx_buf = &c->next_buf;
506 char *eh = c->next_eh;
507 int data_length, hdr_length, dont_merge;
508 unsigned th_seq, pkt_length;
509 struct tcphdr *th;
510 unsigned now;
511
512 if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
513 struct ip *iph = c->next_nh;
514 th = (struct tcphdr *)(iph + 1);
515 pkt_length = ntohs(iph->ip_len) + (char *) iph - eh;
516 } else {
517 struct ip6_hdr *iph = c->next_nh;
518 th = (struct tcphdr *)(iph + 1);
519 pkt_length = ntohs(iph->ip6_plen) + (char *) th - eh;
520 }
521
522 hdr_length = (char *) th + th->th_off * 4 - eh;
523 data_length = (min(pkt_length, rx_buf->size - rxq->sc->rx_prefix_size) -
524 hdr_length);
525 th_seq = ntohl(th->th_seq);
526 dont_merge = ((data_length <= 0)
527 | (th->th_flags & (TH_URG | TH_SYN | TH_RST | TH_FIN)));
528
529 /* Check for options other than aligned timestamp. */
530 if (th->th_off != 5) {
531 const uint32_t *opt_ptr = (const uint32_t *) (th + 1);
532 if (th->th_off == 8 &&
533 opt_ptr[0] == ntohl((TCPOPT_NOP << 24) |
534 (TCPOPT_NOP << 16) |
535 (TCPOPT_TIMESTAMP << 8) |
536 TCPOLEN_TIMESTAMP)) {
537 /* timestamp option -- okay */
538 } else {
539 dont_merge = 1;
540 }
541 }
542
543 if (__predict_false(th_seq != c->next_seq)) {
544 /* Out-of-order, so start counting again. */
545 if (c->mbuf != NULL)
546 sfxge_lro_deliver(&rxq->lro, c);
547 c->n_in_order_pkts -= lro_loss_packets;
548 c->next_seq = th_seq + data_length;
549 ++rxq->lro.n_misorder;
550 goto deliver_buf_out;
551 }
552 c->next_seq = th_seq + data_length;
553
554 now = ticks;
555 if (now - c->last_pkt_ticks > lro_idle_ticks) {
556 ++rxq->lro.n_drop_idle;
557 if (c->mbuf != NULL)
558 sfxge_lro_deliver(&rxq->lro, c);
559 sfxge_lro_drop(rxq, c);
560 return (0);
561 }
562 c->last_pkt_ticks = ticks;
563
564 if (c->n_in_order_pkts < lro_slow_start_packets) {
565 /* May be in slow-start, so don't merge. */
566 ++rxq->lro.n_slow_start;
567 ++c->n_in_order_pkts;
568 goto deliver_buf_out;
569 }
570
571 if (__predict_false(dont_merge)) {
572 if (c->mbuf != NULL)
573 sfxge_lro_deliver(&rxq->lro, c);
574 if (th->th_flags & (TH_FIN | TH_RST)) {
575 ++rxq->lro.n_drop_closed;
576 sfxge_lro_drop(rxq, c);
577 return (0);
578 }
579 goto deliver_buf_out;
580 }
581
582 rx_buf->mbuf->m_data += rxq->sc->rx_prefix_size;
583
584 if (__predict_true(c->mbuf != NULL)) {
585 /* Remove headers and any padding */
586 rx_buf->mbuf->m_data += hdr_length;
587 rx_buf->mbuf->m_len = data_length;
588
589 sfxge_lro_merge(&rxq->lro, c, rx_buf->mbuf, th);
590 } else {
591 /* Remove any padding */
592 rx_buf->mbuf->m_len = pkt_length;
593
594 sfxge_lro_start(&rxq->lro, c, rx_buf->mbuf, c->next_nh, th);
595 }
596
597 rx_buf->mbuf = NULL;
598 return (1);
599
600 deliver_buf_out:
601 sfxge_rx_deliver(rxq->sc, rx_buf);
602 return (1);
603 }
604
sfxge_lro_new_conn(struct sfxge_lro_state * st,uint32_t conn_hash,uint16_t l2_id,void * nh,struct tcphdr * th)605 static void sfxge_lro_new_conn(struct sfxge_lro_state *st, uint32_t conn_hash,
606 uint16_t l2_id, void *nh, struct tcphdr *th)
607 {
608 unsigned bucket = conn_hash & st->conns_mask;
609 struct sfxge_lro_conn *c;
610
611 if (st->conns_n[bucket] >= lro_chain_max) {
612 ++st->n_too_many;
613 return;
614 }
615
616 if (!TAILQ_EMPTY(&st->free_conns)) {
617 c = TAILQ_FIRST(&st->free_conns);
618 TAILQ_REMOVE(&st->free_conns, c, link);
619 } else {
620 c = malloc(sizeof(*c), M_SFXGE, M_NOWAIT);
621 if (c == NULL)
622 return;
623 c->mbuf = NULL;
624 c->next_buf.mbuf = NULL;
625 }
626
627 /* Create the connection tracking data */
628 ++st->conns_n[bucket];
629 TAILQ_INSERT_HEAD(&st->conns[bucket], c, link);
630 c->l2_id = l2_id;
631 c->conn_hash = conn_hash;
632 c->source = th->th_sport;
633 c->dest = th->th_dport;
634 c->n_in_order_pkts = 0;
635 c->last_pkt_ticks = *(volatile int *)&ticks;
636 c->delivered = 0;
637 ++st->n_new_stream;
638 /* NB. We don't initialise c->next_seq, and it doesn't matter what
639 * value it has. Most likely the next packet received for this
640 * connection will not match -- no harm done.
641 */
642 }
643
644 /* Process mbuf and decide whether to dispatch it to the stack now or
645 * later.
646 */
647 static void
sfxge_lro(struct sfxge_rxq * rxq,struct sfxge_rx_sw_desc * rx_buf)648 sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf)
649 {
650 struct sfxge_softc *sc = rxq->sc;
651 struct mbuf *m = rx_buf->mbuf;
652 struct ether_header *eh;
653 struct sfxge_lro_conn *c;
654 uint16_t l2_id;
655 uint16_t l3_proto;
656 void *nh;
657 struct tcphdr *th;
658 uint32_t conn_hash;
659 unsigned bucket;
660
661 /* Get the hardware hash */
662 conn_hash = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ,
663 mtod(m, uint8_t *));
664
665 eh = (struct ether_header *)(m->m_data + sc->rx_prefix_size);
666 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
667 struct ether_vlan_header *veh = (struct ether_vlan_header *)eh;
668 l2_id = EVL_VLANOFTAG(ntohs(veh->evl_tag)) |
669 SFXGE_LRO_L2_ID_VLAN;
670 l3_proto = veh->evl_proto;
671 nh = veh + 1;
672 } else {
673 l2_id = 0;
674 l3_proto = eh->ether_type;
675 nh = eh + 1;
676 }
677
678 /* Check whether this is a suitable packet (unfragmented
679 * TCP/IPv4 or TCP/IPv6). If so, find the TCP header and
680 * length, and compute a hash if necessary. If not, return.
681 */
682 if (l3_proto == htons(ETHERTYPE_IP)) {
683 struct ip *iph = nh;
684
685 KASSERT(iph->ip_p == IPPROTO_TCP,
686 ("IPv4 protocol is not TCP, but packet marker is set"));
687 if ((iph->ip_hl - (sizeof(*iph) >> 2u)) |
688 (iph->ip_off & htons(IP_MF | IP_OFFMASK)))
689 goto deliver_now;
690 th = (struct tcphdr *)(iph + 1);
691 } else if (l3_proto == htons(ETHERTYPE_IPV6)) {
692 struct ip6_hdr *iph = nh;
693
694 KASSERT(iph->ip6_nxt == IPPROTO_TCP,
695 ("IPv6 next header is not TCP, but packet marker is set"));
696 l2_id |= SFXGE_LRO_L2_ID_IPV6;
697 th = (struct tcphdr *)(iph + 1);
698 } else {
699 goto deliver_now;
700 }
701
702 bucket = conn_hash & rxq->lro.conns_mask;
703
704 TAILQ_FOREACH(c, &rxq->lro.conns[bucket], link) {
705 if ((c->l2_id - l2_id) | (c->conn_hash - conn_hash))
706 continue;
707 if ((c->source - th->th_sport) | (c->dest - th->th_dport))
708 continue;
709 if (c->mbuf != NULL) {
710 if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) {
711 struct ip *c_iph, *iph = nh;
712 c_iph = c->nh;
713 if ((c_iph->ip_src.s_addr - iph->ip_src.s_addr) |
714 (c_iph->ip_dst.s_addr - iph->ip_dst.s_addr))
715 continue;
716 } else {
717 struct ip6_hdr *c_iph, *iph = nh;
718 c_iph = c->nh;
719 if (ipv6_addr_cmp(&c_iph->ip6_src, &iph->ip6_src) |
720 ipv6_addr_cmp(&c_iph->ip6_dst, &iph->ip6_dst))
721 continue;
722 }
723 }
724
725 /* Re-insert at head of list to reduce lookup time. */
726 TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link);
727 TAILQ_INSERT_HEAD(&rxq->lro.conns[bucket], c, link);
728
729 if (c->next_buf.mbuf != NULL) {
730 if (!sfxge_lro_try_merge(rxq, c))
731 goto deliver_now;
732 } else {
733 LIST_INSERT_HEAD(&rxq->lro.active_conns, c,
734 active_link);
735 }
736 c->next_buf = *rx_buf;
737 c->next_eh = eh;
738 c->next_nh = nh;
739
740 rx_buf->mbuf = NULL;
741 rx_buf->flags = EFX_DISCARD;
742 return;
743 }
744
745 sfxge_lro_new_conn(&rxq->lro, conn_hash, l2_id, nh, th);
746 deliver_now:
747 sfxge_rx_deliver(sc, rx_buf);
748 }
749
sfxge_lro_end_of_burst(struct sfxge_rxq * rxq)750 static void sfxge_lro_end_of_burst(struct sfxge_rxq *rxq)
751 {
752 struct sfxge_lro_state *st = &rxq->lro;
753 struct sfxge_lro_conn *c;
754 unsigned t;
755
756 while (!LIST_EMPTY(&st->active_conns)) {
757 c = LIST_FIRST(&st->active_conns);
758 if (!c->delivered && c->mbuf != NULL)
759 sfxge_lro_deliver(st, c);
760 if (sfxge_lro_try_merge(rxq, c)) {
761 if (c->mbuf != NULL)
762 sfxge_lro_deliver(st, c);
763 LIST_REMOVE(c, active_link);
764 }
765 c->delivered = 0;
766 }
767
768 t = *(volatile int *)&ticks;
769 if (__predict_false(t != st->last_purge_ticks))
770 sfxge_lro_purge_idle(rxq, t);
771 }
772
773 #else /* !SFXGE_LRO */
774
775 static void
sfxge_lro(struct sfxge_rxq * rxq,struct sfxge_rx_sw_desc * rx_buf)776 sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf)
777 {
778 }
779
780 static void
sfxge_lro_end_of_burst(struct sfxge_rxq * rxq)781 sfxge_lro_end_of_burst(struct sfxge_rxq *rxq)
782 {
783 }
784
785 #endif /* SFXGE_LRO */
786
787 void
sfxge_rx_qcomplete(struct sfxge_rxq * rxq,boolean_t eop)788 sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop)
789 {
790 struct sfxge_softc *sc = rxq->sc;
791 int if_capenable = sc->ifnet->if_capenable;
792 int lro_enabled = if_capenable & IFCAP_LRO;
793 unsigned int index;
794 struct sfxge_evq *evq;
795 unsigned int completed;
796 unsigned int level;
797 struct mbuf *m;
798 struct sfxge_rx_sw_desc *prev = NULL;
799
800 index = rxq->index;
801 evq = sc->evq[index];
802
803 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
804
805 completed = rxq->completed;
806 while (completed != rxq->pending) {
807 unsigned int id;
808 struct sfxge_rx_sw_desc *rx_desc;
809
810 id = completed++ & rxq->ptr_mask;
811 rx_desc = &rxq->queue[id];
812 m = rx_desc->mbuf;
813
814 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
815 goto discard;
816
817 if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
818 goto discard;
819
820 prefetch_read_many(mtod(m, caddr_t));
821
822 switch (rx_desc->flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) {
823 case EFX_PKT_IPV4:
824 if (~if_capenable & IFCAP_RXCSUM)
825 rx_desc->flags &=
826 ~(EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP);
827 break;
828 case EFX_PKT_IPV6:
829 if (~if_capenable & IFCAP_RXCSUM_IPV6)
830 rx_desc->flags &= ~EFX_CKSUM_TCPUDP;
831 break;
832 case 0:
833 /* Check for loopback packets */
834 {
835 struct ether_header *etherhp;
836
837 /*LINTED*/
838 etherhp = mtod(m, struct ether_header *);
839
840 if (etherhp->ether_type ==
841 htons(SFXGE_ETHERTYPE_LOOPBACK)) {
842 EFSYS_PROBE(loopback);
843
844 rxq->loopback++;
845 goto discard;
846 }
847 }
848 break;
849 default:
850 KASSERT(B_FALSE,
851 ("Rx descriptor with both IPv4 and IPv6 flags"));
852 goto discard;
853 }
854
855 /* Pass packet up the stack or into LRO (pipelined) */
856 if (prev != NULL) {
857 if (lro_enabled &&
858 ((prev->flags & (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)) ==
859 (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)))
860 sfxge_lro(rxq, prev);
861 else
862 sfxge_rx_deliver(sc, prev);
863 }
864 prev = rx_desc;
865 continue;
866
867 discard:
868 /* Return the packet to the pool */
869 m_free(m);
870 rx_desc->mbuf = NULL;
871 }
872 rxq->completed = completed;
873
874 level = rxq->added - rxq->completed;
875
876 /* Pass last packet up the stack or into LRO */
877 if (prev != NULL) {
878 if (lro_enabled &&
879 ((prev->flags & (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)) ==
880 (EFX_PKT_TCP | EFX_CKSUM_TCPUDP)))
881 sfxge_lro(rxq, prev);
882 else
883 sfxge_rx_deliver(sc, prev);
884 }
885
886 /*
887 * If there are any pending flows and this is the end of the
888 * poll then they must be completed.
889 */
890 if (eop)
891 sfxge_lro_end_of_burst(rxq);
892
893 /* Top up the queue if necessary */
894 if (level < rxq->refill_threshold)
895 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_FALSE);
896 }
897
898 static void
sfxge_rx_qstop(struct sfxge_softc * sc,unsigned int index)899 sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index)
900 {
901 struct sfxge_rxq *rxq;
902 struct sfxge_evq *evq;
903 unsigned int count;
904
905 rxq = sc->rxq[index];
906 evq = sc->evq[index];
907
908 SFXGE_EVQ_LOCK(evq);
909
910 KASSERT(rxq->init_state == SFXGE_RXQ_STARTED,
911 ("rxq not started"));
912
913 rxq->init_state = SFXGE_RXQ_INITIALIZED;
914
915 callout_stop(&rxq->refill_callout);
916
917 again:
918 rxq->flush_state = SFXGE_FLUSH_PENDING;
919
920 /* Flush the receive queue */
921 efx_rx_qflush(rxq->common);
922
923 SFXGE_EVQ_UNLOCK(evq);
924
925 count = 0;
926 do {
927 /* Spin for 100 ms */
928 DELAY(100000);
929
930 if (rxq->flush_state != SFXGE_FLUSH_PENDING)
931 break;
932
933 } while (++count < 20);
934
935 SFXGE_EVQ_LOCK(evq);
936
937 if (rxq->flush_state == SFXGE_FLUSH_FAILED)
938 goto again;
939
940 rxq->flush_state = SFXGE_FLUSH_DONE;
941
942 rxq->pending = rxq->added;
943 sfxge_rx_qcomplete(rxq, B_TRUE);
944
945 KASSERT(rxq->completed == rxq->pending,
946 ("rxq->completed != rxq->pending"));
947
948 rxq->added = 0;
949 rxq->pending = 0;
950 rxq->completed = 0;
951 rxq->loopback = 0;
952
953 /* Destroy the common code receive queue. */
954 efx_rx_qdestroy(rxq->common);
955
956 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
957 EFX_RXQ_NBUFS(sc->rxq_entries));
958
959 SFXGE_EVQ_UNLOCK(evq);
960 }
961
962 static int
sfxge_rx_qstart(struct sfxge_softc * sc,unsigned int index)963 sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index)
964 {
965 struct sfxge_rxq *rxq;
966 efsys_mem_t *esmp;
967 struct sfxge_evq *evq;
968 int rc;
969
970 rxq = sc->rxq[index];
971 esmp = &rxq->mem;
972 evq = sc->evq[index];
973
974 KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
975 ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
976 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
977 ("evq->init_state != SFXGE_EVQ_STARTED"));
978
979 /* Program the buffer table. */
980 if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp,
981 EFX_RXQ_NBUFS(sc->rxq_entries))) != 0)
982 return (rc);
983
984 /* Create the common code receive queue. */
985 if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT,
986 esmp, sc->rxq_entries, rxq->buf_base_id, evq->common,
987 &rxq->common)) != 0)
988 goto fail;
989
990 SFXGE_EVQ_LOCK(evq);
991
992 /* Enable the receive queue. */
993 efx_rx_qenable(rxq->common);
994
995 rxq->init_state = SFXGE_RXQ_STARTED;
996
997 /* Try to fill the queue from the pool. */
998 sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(sc->rxq_entries), B_FALSE);
999
1000 SFXGE_EVQ_UNLOCK(evq);
1001
1002 return (0);
1003
1004 fail:
1005 efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id,
1006 EFX_RXQ_NBUFS(sc->rxq_entries));
1007 return (rc);
1008 }
1009
1010 void
sfxge_rx_stop(struct sfxge_softc * sc)1011 sfxge_rx_stop(struct sfxge_softc *sc)
1012 {
1013 int index;
1014
1015 /* Stop the receive queue(s) */
1016 index = sc->rxq_count;
1017 while (--index >= 0)
1018 sfxge_rx_qstop(sc, index);
1019
1020 sc->rx_prefix_size = 0;
1021 sc->rx_buffer_size = 0;
1022
1023 efx_rx_fini(sc->enp);
1024 }
1025
1026 int
sfxge_rx_start(struct sfxge_softc * sc)1027 sfxge_rx_start(struct sfxge_softc *sc)
1028 {
1029 struct sfxge_intr *intr;
1030 int index;
1031 int rc;
1032
1033 intr = &sc->intr;
1034
1035 /* Initialize the common code receive module. */
1036 if ((rc = efx_rx_init(sc->enp)) != 0)
1037 return (rc);
1038
1039 /* Calculate the receive packet buffer size. */
1040 sc->rx_prefix_size = EFX_RX_PREFIX_SIZE;
1041 sc->rx_buffer_size = (EFX_MAC_PDU(sc->ifnet->if_mtu) +
1042 sc->rx_prefix_size);
1043
1044 /* Select zone for packet buffers */
1045 if (sc->rx_buffer_size <= MCLBYTES)
1046 sc->rx_buffer_zone = zone_clust;
1047 else if (sc->rx_buffer_size <= MJUMPAGESIZE)
1048 sc->rx_buffer_zone = zone_jumbop;
1049 else if (sc->rx_buffer_size <= MJUM9BYTES)
1050 sc->rx_buffer_zone = zone_jumbo9;
1051 else
1052 sc->rx_buffer_zone = zone_jumbo16;
1053
1054 /*
1055 * Set up the scale table. Enable all hash types and hash insertion.
1056 */
1057 for (index = 0; index < SFXGE_RX_SCALE_MAX; index++)
1058 sc->rx_indir_table[index] = index % sc->rxq_count;
1059 if ((rc = efx_rx_scale_tbl_set(sc->enp, sc->rx_indir_table,
1060 SFXGE_RX_SCALE_MAX)) != 0)
1061 goto fail;
1062 (void)efx_rx_scale_mode_set(sc->enp, EFX_RX_HASHALG_TOEPLITZ,
1063 (1 << EFX_RX_HASH_IPV4) | (1 << EFX_RX_HASH_TCPIPV4) |
1064 (1 << EFX_RX_HASH_IPV6) | (1 << EFX_RX_HASH_TCPIPV6), B_TRUE);
1065
1066 if ((rc = efx_rx_scale_toeplitz_ipv4_key_set(sc->enp, toep_key,
1067 sizeof(toep_key))) != 0)
1068 goto fail;
1069
1070 /* Start the receive queue(s). */
1071 for (index = 0; index < sc->rxq_count; index++) {
1072 if ((rc = sfxge_rx_qstart(sc, index)) != 0)
1073 goto fail2;
1074 }
1075
1076 return (0);
1077
1078 fail2:
1079 while (--index >= 0)
1080 sfxge_rx_qstop(sc, index);
1081
1082 fail:
1083 efx_rx_fini(sc->enp);
1084
1085 return (rc);
1086 }
1087
1088 #ifdef SFXGE_LRO
1089
sfxge_lro_init(struct sfxge_rxq * rxq)1090 static void sfxge_lro_init(struct sfxge_rxq *rxq)
1091 {
1092 struct sfxge_lro_state *st = &rxq->lro;
1093 unsigned i;
1094
1095 st->conns_mask = lro_table_size - 1;
1096 KASSERT(!((st->conns_mask + 1) & st->conns_mask),
1097 ("lro_table_size must be a power of 2"));
1098 st->sc = rxq->sc;
1099 st->conns = malloc((st->conns_mask + 1) * sizeof(st->conns[0]),
1100 M_SFXGE, M_WAITOK);
1101 st->conns_n = malloc((st->conns_mask + 1) * sizeof(st->conns_n[0]),
1102 M_SFXGE, M_WAITOK);
1103 for (i = 0; i <= st->conns_mask; ++i) {
1104 TAILQ_INIT(&st->conns[i]);
1105 st->conns_n[i] = 0;
1106 }
1107 LIST_INIT(&st->active_conns);
1108 TAILQ_INIT(&st->free_conns);
1109 }
1110
sfxge_lro_fini(struct sfxge_rxq * rxq)1111 static void sfxge_lro_fini(struct sfxge_rxq *rxq)
1112 {
1113 struct sfxge_lro_state *st = &rxq->lro;
1114 struct sfxge_lro_conn *c;
1115 unsigned i;
1116
1117 /* Return cleanly if sfxge_lro_init() has not been called. */
1118 if (st->conns == NULL)
1119 return;
1120
1121 KASSERT(LIST_EMPTY(&st->active_conns), ("found active connections"));
1122
1123 for (i = 0; i <= st->conns_mask; ++i) {
1124 while (!TAILQ_EMPTY(&st->conns[i])) {
1125 c = TAILQ_LAST(&st->conns[i], sfxge_lro_tailq);
1126 sfxge_lro_drop(rxq, c);
1127 }
1128 }
1129
1130 while (!TAILQ_EMPTY(&st->free_conns)) {
1131 c = TAILQ_FIRST(&st->free_conns);
1132 TAILQ_REMOVE(&st->free_conns, c, link);
1133 KASSERT(!c->mbuf, ("found orphaned mbuf"));
1134 free(c, M_SFXGE);
1135 }
1136
1137 free(st->conns_n, M_SFXGE);
1138 free(st->conns, M_SFXGE);
1139 st->conns = NULL;
1140 }
1141
1142 #else
1143
1144 static void
sfxge_lro_init(struct sfxge_rxq * rxq)1145 sfxge_lro_init(struct sfxge_rxq *rxq)
1146 {
1147 }
1148
1149 static void
sfxge_lro_fini(struct sfxge_rxq * rxq)1150 sfxge_lro_fini(struct sfxge_rxq *rxq)
1151 {
1152 }
1153
1154 #endif /* SFXGE_LRO */
1155
1156 static void
sfxge_rx_qfini(struct sfxge_softc * sc,unsigned int index)1157 sfxge_rx_qfini(struct sfxge_softc *sc, unsigned int index)
1158 {
1159 struct sfxge_rxq *rxq;
1160
1161 rxq = sc->rxq[index];
1162
1163 KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED,
1164 ("rxq->init_state != SFXGE_RXQ_INITIALIZED"));
1165
1166 /* Free the context array and the flow table. */
1167 free(rxq->queue, M_SFXGE);
1168 sfxge_lro_fini(rxq);
1169
1170 /* Release DMA memory. */
1171 sfxge_dma_free(&rxq->mem);
1172
1173 sc->rxq[index] = NULL;
1174
1175 free(rxq, M_SFXGE);
1176 }
1177
1178 static int
sfxge_rx_qinit(struct sfxge_softc * sc,unsigned int index)1179 sfxge_rx_qinit(struct sfxge_softc *sc, unsigned int index)
1180 {
1181 struct sfxge_rxq *rxq;
1182 struct sfxge_evq *evq;
1183 efsys_mem_t *esmp;
1184 int rc;
1185
1186 KASSERT(index < sc->rxq_count, ("index >= %d", sc->rxq_count));
1187
1188 rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK);
1189 rxq->sc = sc;
1190 rxq->index = index;
1191 rxq->entries = sc->rxq_entries;
1192 rxq->ptr_mask = rxq->entries - 1;
1193 rxq->refill_threshold = RX_REFILL_THRESHOLD(rxq->entries);
1194
1195 sc->rxq[index] = rxq;
1196 esmp = &rxq->mem;
1197
1198 evq = sc->evq[index];
1199
1200 /* Allocate and zero DMA space. */
1201 if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(sc->rxq_entries), esmp)) != 0)
1202 return (rc);
1203
1204 /* Allocate buffer table entries. */
1205 sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(sc->rxq_entries),
1206 &rxq->buf_base_id);
1207
1208 /* Allocate the context array and the flow table. */
1209 rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * sc->rxq_entries,
1210 M_SFXGE, M_WAITOK | M_ZERO);
1211 sfxge_lro_init(rxq);
1212
1213 callout_init(&rxq->refill_callout, B_TRUE);
1214
1215 rxq->init_state = SFXGE_RXQ_INITIALIZED;
1216
1217 return (0);
1218 }
1219
1220 static const struct {
1221 const char *name;
1222 size_t offset;
1223 } sfxge_rx_stats[] = {
1224 #define SFXGE_RX_STAT(name, member) \
1225 { #name, offsetof(struct sfxge_rxq, member) }
1226 #ifdef SFXGE_LRO
1227 SFXGE_RX_STAT(lro_merges, lro.n_merges),
1228 SFXGE_RX_STAT(lro_bursts, lro.n_bursts),
1229 SFXGE_RX_STAT(lro_slow_start, lro.n_slow_start),
1230 SFXGE_RX_STAT(lro_misorder, lro.n_misorder),
1231 SFXGE_RX_STAT(lro_too_many, lro.n_too_many),
1232 SFXGE_RX_STAT(lro_new_stream, lro.n_new_stream),
1233 SFXGE_RX_STAT(lro_drop_idle, lro.n_drop_idle),
1234 SFXGE_RX_STAT(lro_drop_closed, lro.n_drop_closed)
1235 #endif
1236 };
1237
1238 static int
sfxge_rx_stat_handler(SYSCTL_HANDLER_ARGS)1239 sfxge_rx_stat_handler(SYSCTL_HANDLER_ARGS)
1240 {
1241 struct sfxge_softc *sc = arg1;
1242 unsigned int id = arg2;
1243 unsigned int sum, index;
1244
1245 /* Sum across all RX queues */
1246 sum = 0;
1247 for (index = 0; index < sc->rxq_count; index++)
1248 sum += *(unsigned int *)((caddr_t)sc->rxq[index] +
1249 sfxge_rx_stats[id].offset);
1250
1251 return (SYSCTL_OUT(req, &sum, sizeof(sum)));
1252 }
1253
1254 static void
sfxge_rx_stat_init(struct sfxge_softc * sc)1255 sfxge_rx_stat_init(struct sfxge_softc *sc)
1256 {
1257 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
1258 struct sysctl_oid_list *stat_list;
1259 unsigned int id;
1260
1261 stat_list = SYSCTL_CHILDREN(sc->stats_node);
1262
1263 for (id = 0; id < nitems(sfxge_rx_stats); id++) {
1264 SYSCTL_ADD_PROC(
1265 ctx, stat_list,
1266 OID_AUTO, sfxge_rx_stats[id].name,
1267 CTLTYPE_UINT|CTLFLAG_RD,
1268 sc, id, sfxge_rx_stat_handler, "IU",
1269 "");
1270 }
1271 }
1272
1273 void
sfxge_rx_fini(struct sfxge_softc * sc)1274 sfxge_rx_fini(struct sfxge_softc *sc)
1275 {
1276 int index;
1277
1278 index = sc->rxq_count;
1279 while (--index >= 0)
1280 sfxge_rx_qfini(sc, index);
1281
1282 sc->rxq_count = 0;
1283 }
1284
1285 int
sfxge_rx_init(struct sfxge_softc * sc)1286 sfxge_rx_init(struct sfxge_softc *sc)
1287 {
1288 struct sfxge_intr *intr;
1289 int index;
1290 int rc;
1291
1292 #ifdef SFXGE_LRO
1293 if (!ISP2(lro_table_size)) {
1294 log(LOG_ERR, "%s=%u must be power of 2",
1295 SFXGE_LRO_PARAM(table_size), lro_table_size);
1296 rc = EINVAL;
1297 goto fail_lro_table_size;
1298 }
1299
1300 if (lro_idle_ticks == 0)
1301 lro_idle_ticks = hz / 10 + 1; /* 100 ms */
1302 #endif
1303
1304 intr = &sc->intr;
1305
1306 sc->rxq_count = intr->n_alloc;
1307
1308 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
1309 ("intr->state != SFXGE_INTR_INITIALIZED"));
1310
1311 /* Initialize the receive queue(s) - one per interrupt. */
1312 for (index = 0; index < sc->rxq_count; index++) {
1313 if ((rc = sfxge_rx_qinit(sc, index)) != 0)
1314 goto fail;
1315 }
1316
1317 sfxge_rx_stat_init(sc);
1318
1319 return (0);
1320
1321 fail:
1322 /* Tear down the receive queue(s). */
1323 while (--index >= 0)
1324 sfxge_rx_qfini(sc, index);
1325
1326 sc->rxq_count = 0;
1327
1328 #ifdef SFXGE_LRO
1329 fail_lro_table_size:
1330 #endif
1331 return (rc);
1332 }
1333