1 /*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31
32 #ifdef TCP_OFFLOAD
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/bus.h>
38 #include <sys/fnv_hash.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/rwlock.h>
42 #include <sys/socket.h>
43 #include <sys/sbuf.h>
44 #include <sys/taskqueue.h>
45 #include <net/if.h>
46 #include <net/if_types.h>
47 #include <net/ethernet.h>
48 #include <net/if_vlan_var.h>
49 #include <net/route.h>
50 #include <netinet/in.h>
51 #include <netinet/toecore.h>
52
53 #include "common/common.h"
54 #include "common/t4_msg.h"
55 #include "tom/t4_tom_l2t.h"
56 #include "tom/t4_tom.h"
57
58 #define VLAN_NONE 0xfff
59
60 static inline void
l2t_hold(struct l2t_data * d,struct l2t_entry * e)61 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
62 {
63
64 if (atomic_fetchadd_int(&e->refcnt, 1) == 0) /* 0 -> 1 transition */
65 atomic_subtract_int(&d->nfree, 1);
66 }
67
68 static inline u_int
l2_hash(struct l2t_data * d,const struct sockaddr * sa,int ifindex)69 l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
70 {
71 u_int hash, half = d->l2t_size / 2, start = 0;
72 const void *key;
73 size_t len;
74
75 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
76 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
77 sa->sa_family));
78
79 if (sa->sa_family == AF_INET) {
80 const struct sockaddr_in *sin = (const void *)sa;
81
82 key = &sin->sin_addr;
83 len = sizeof(sin->sin_addr);
84 } else {
85 const struct sockaddr_in6 *sin6 = (const void *)sa;
86
87 key = &sin6->sin6_addr;
88 len = sizeof(sin6->sin6_addr);
89 start = half;
90 }
91
92 hash = fnv_32_buf(key, len, FNV1_32_INIT);
93 hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
94 hash %= half;
95
96 return (hash + start);
97 }
98
99 static inline int
l2_cmp(const struct sockaddr * sa,struct l2t_entry * e)100 l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
101 {
102
103 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
104 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
105 sa->sa_family));
106
107 if (sa->sa_family == AF_INET) {
108 const struct sockaddr_in *sin = (const void *)sa;
109
110 return (e->addr[0] != sin->sin_addr.s_addr);
111 } else {
112 const struct sockaddr_in6 *sin6 = (const void *)sa;
113
114 return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
115 }
116 }
117
118 static inline void
l2_store(const struct sockaddr * sa,struct l2t_entry * e)119 l2_store(const struct sockaddr *sa, struct l2t_entry *e)
120 {
121
122 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
123 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
124 sa->sa_family));
125
126 if (sa->sa_family == AF_INET) {
127 const struct sockaddr_in *sin = (const void *)sa;
128
129 e->addr[0] = sin->sin_addr.s_addr;
130 e->ipv6 = 0;
131 } else {
132 const struct sockaddr_in6 *sin6 = (const void *)sa;
133
134 memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
135 e->ipv6 = 1;
136 }
137 }
138
139 /*
140 * Add a WR to an L2T entry's queue of work requests awaiting resolution.
141 * Must be called with the entry's lock held.
142 */
143 static inline void
arpq_enqueue(struct l2t_entry * e,struct wrqe * wr)144 arpq_enqueue(struct l2t_entry *e, struct wrqe *wr)
145 {
146 mtx_assert(&e->lock, MA_OWNED);
147
148 STAILQ_INSERT_TAIL(&e->wr_list, wr, link);
149 }
150
151 static inline void
send_pending(struct adapter * sc,struct l2t_entry * e)152 send_pending(struct adapter *sc, struct l2t_entry *e)
153 {
154 struct wrqe *wr;
155
156 mtx_assert(&e->lock, MA_OWNED);
157
158 while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
159 STAILQ_REMOVE_HEAD(&e->wr_list, link);
160 t4_wrq_tx(sc, wr);
161 }
162 }
163
164 static void
resolution_failed(struct adapter * sc,struct l2t_entry * e)165 resolution_failed(struct adapter *sc, struct l2t_entry *e)
166 {
167 struct tom_data *td = sc->tom_softc;
168
169 mtx_assert(&e->lock, MA_OWNED);
170
171 mtx_lock(&td->unsent_wr_lock);
172 STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list);
173 mtx_unlock(&td->unsent_wr_lock);
174
175 taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources);
176 }
177
178 static void
update_entry(struct adapter * sc,struct l2t_entry * e,uint8_t * lladdr,uint16_t vtag)179 update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr,
180 uint16_t vtag)
181 {
182
183 mtx_assert(&e->lock, MA_OWNED);
184
185 /*
186 * The entry may be in active use (e->refcount > 0) or not. We update
187 * it even when it's not as this simplifies the case where we decide to
188 * reuse the entry later.
189 */
190
191 if (lladdr == NULL &&
192 (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) {
193 /*
194 * Never got a valid L2 address for this one. Just mark it as
195 * failed instead of removing it from the hash (for which we'd
196 * need to wlock the table).
197 */
198 e->state = L2T_STATE_FAILED;
199 resolution_failed(sc, e);
200 return;
201
202 } else if (lladdr == NULL) {
203
204 /* Valid or already-stale entry was deleted (or expired) */
205
206 KASSERT(e->state == L2T_STATE_VALID ||
207 e->state == L2T_STATE_STALE,
208 ("%s: lladdr NULL, state %d", __func__, e->state));
209
210 e->state = L2T_STATE_STALE;
211
212 } else {
213
214 if (e->state == L2T_STATE_RESOLVING ||
215 e->state == L2T_STATE_FAILED ||
216 memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) {
217
218 /* unresolved -> resolved; or dmac changed */
219
220 memcpy(e->dmac, lladdr, ETHER_ADDR_LEN);
221 e->vlan = vtag;
222 t4_write_l2e(sc, e, 1);
223 }
224 e->state = L2T_STATE_VALID;
225 }
226 }
227
228 static int
resolve_entry(struct adapter * sc,struct l2t_entry * e)229 resolve_entry(struct adapter *sc, struct l2t_entry *e)
230 {
231 struct tom_data *td = sc->tom_softc;
232 struct toedev *tod = &td->tod;
233 struct sockaddr_in sin = {0};
234 struct sockaddr_in6 sin6 = {0};
235 struct sockaddr *sa;
236 uint8_t dmac[ETHER_ADDR_LEN];
237 uint16_t vtag = VLAN_NONE;
238 int rc;
239
240 if (e->ipv6 == 0) {
241 sin.sin_family = AF_INET;
242 sin.sin_len = sizeof(struct sockaddr_in);
243 sin.sin_addr.s_addr = e->addr[0];
244 sa = (void *)&sin;
245 } else {
246 sin6.sin6_family = AF_INET6;
247 sin6.sin6_len = sizeof(struct sockaddr_in6);
248 memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
249 sa = (void *)&sin6;
250 }
251
252 rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
253 if (rc == EWOULDBLOCK)
254 return (rc);
255
256 mtx_lock(&e->lock);
257 update_entry(sc, e, rc == 0 ? dmac : NULL, vtag);
258 mtx_unlock(&e->lock);
259
260 return (rc);
261 }
262
263 int
t4_l2t_send_slow(struct adapter * sc,struct wrqe * wr,struct l2t_entry * e)264 t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
265 {
266
267 again:
268 switch (e->state) {
269 case L2T_STATE_STALE: /* entry is stale, kick off revalidation */
270
271 if (resolve_entry(sc, e) != EWOULDBLOCK)
272 goto again; /* entry updated, re-examine state */
273
274 /* Fall through */
275
276 case L2T_STATE_VALID: /* fast-path, send the packet on */
277
278 t4_wrq_tx(sc, wr);
279 return (0);
280
281 case L2T_STATE_RESOLVING:
282 case L2T_STATE_SYNC_WRITE:
283
284 mtx_lock(&e->lock);
285 if (e->state != L2T_STATE_SYNC_WRITE &&
286 e->state != L2T_STATE_RESOLVING) {
287 /* state changed by the time we got here */
288 mtx_unlock(&e->lock);
289 goto again;
290 }
291 arpq_enqueue(e, wr);
292 mtx_unlock(&e->lock);
293
294 if (resolve_entry(sc, e) == EWOULDBLOCK)
295 break;
296
297 mtx_lock(&e->lock);
298 if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list))
299 send_pending(sc, e);
300 if (e->state == L2T_STATE_FAILED)
301 resolution_failed(sc, e);
302 mtx_unlock(&e->lock);
303 break;
304
305 case L2T_STATE_FAILED:
306 return (EHOSTUNREACH);
307 }
308
309 return (0);
310 }
311
312 /*
313 * Called when an L2T entry has no more users. The entry is left in the hash
314 * table since it is likely to be reused but we also bump nfree to indicate
315 * that the entry can be reallocated for a different neighbor. We also drop
316 * the existing neighbor reference in case the neighbor is going away and is
317 * waiting on our reference.
318 *
319 * Because entries can be reallocated to other neighbors once their ref count
320 * drops to 0 we need to take the entry's lock to avoid races with a new
321 * incarnation.
322 */
323
324 static int
do_l2t_write_rpl2(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)325 do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
326 struct mbuf *m)
327 {
328 struct adapter *sc = iq->adapter;
329 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
330 unsigned int tid = GET_TID(rpl);
331 unsigned int idx = tid % L2T_SIZE;
332 int rc;
333
334 rc = do_l2t_write_rpl(iq, rss, m);
335 if (rc != 0)
336 return (rc);
337
338 if (tid & F_SYNC_WR) {
339 struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
340
341 mtx_lock(&e->lock);
342 if (e->state != L2T_STATE_SWITCHING) {
343 send_pending(sc, e);
344 e->state = L2T_STATE_VALID;
345 }
346 mtx_unlock(&e->lock);
347 }
348
349 return (0);
350 }
351
352 void
t4_init_l2t_cpl_handlers(struct adapter * sc)353 t4_init_l2t_cpl_handlers(struct adapter *sc)
354 {
355
356 t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl2);
357 }
358
359 void
t4_uninit_l2t_cpl_handlers(struct adapter * sc)360 t4_uninit_l2t_cpl_handlers(struct adapter *sc)
361 {
362
363 t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
364 }
365
366 /*
367 * The TOE wants an L2 table entry that it can use to reach the next hop over
368 * the specified port. Produce such an entry - create one if needed.
369 *
370 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
371 * top of the real cxgbe interface.
372 */
373 struct l2t_entry *
t4_l2t_get(struct port_info * pi,struct ifnet * ifp,struct sockaddr * sa)374 t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
375 {
376 struct l2t_entry *e;
377 struct l2t_data *d = pi->adapter->l2t;
378 u_int hash, smt_idx = pi->port_id;
379
380 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
381 ("%s: sa %p has unexpected sa_family %d", __func__, sa,
382 sa->sa_family));
383
384 #ifndef VLAN_TAG
385 if (ifp->if_type == IFT_L2VLAN)
386 return (NULL);
387 #endif
388
389 hash = l2_hash(d, sa, ifp->if_index);
390 rw_wlock(&d->lock);
391 for (e = d->l2tab[hash].first; e; e = e->next) {
392 if (l2_cmp(sa, e) == 0 && e->ifp == ifp &&
393 e->smt_idx == smt_idx) {
394 l2t_hold(d, e);
395 goto done;
396 }
397 }
398
399 /* Need to allocate a new entry */
400 e = t4_alloc_l2e(d);
401 if (e) {
402 mtx_lock(&e->lock); /* avoid race with t4_l2t_free */
403 e->next = d->l2tab[hash].first;
404 d->l2tab[hash].first = e;
405
406 e->state = L2T_STATE_RESOLVING;
407 l2_store(sa, e);
408 e->ifp = ifp;
409 e->smt_idx = smt_idx;
410 e->hash = hash;
411 e->lport = pi->lport;
412 atomic_store_rel_int(&e->refcnt, 1);
413 #ifdef VLAN_TAG
414 if (ifp->if_type == IFT_L2VLAN)
415 VLAN_TAG(ifp, &e->vlan);
416 else
417 e->vlan = VLAN_NONE;
418 #endif
419 mtx_unlock(&e->lock);
420 }
421 done:
422 rw_wunlock(&d->lock);
423 return e;
424 }
425
426 /*
427 * Called when the host's ARP layer makes a change to some entry that is loaded
428 * into the HW L2 table.
429 */
430 void
t4_l2_update(struct toedev * tod,struct ifnet * ifp,struct sockaddr * sa,uint8_t * lladdr,uint16_t vtag)431 t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
432 uint8_t *lladdr, uint16_t vtag)
433 {
434 struct adapter *sc = tod->tod_softc;
435 struct l2t_entry *e;
436 struct l2t_data *d = sc->l2t;
437 u_int hash;
438
439 KASSERT(d != NULL, ("%s: no L2 table", __func__));
440
441 hash = l2_hash(d, sa, ifp->if_index);
442 rw_rlock(&d->lock);
443 for (e = d->l2tab[hash].first; e; e = e->next) {
444 if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
445 mtx_lock(&e->lock);
446 if (atomic_load_acq_int(&e->refcnt))
447 goto found;
448 e->state = L2T_STATE_STALE;
449 mtx_unlock(&e->lock);
450 break;
451 }
452 }
453 rw_runlock(&d->lock);
454
455 /*
456 * This is of no interest to us. We've never had an offloaded
457 * connection to this destination, and we aren't attempting one right
458 * now.
459 */
460 return;
461
462 found:
463 rw_runlock(&d->lock);
464
465 KASSERT(e->state != L2T_STATE_UNUSED,
466 ("%s: unused entry in the hash.", __func__));
467
468 update_entry(sc, e, lladdr, vtag);
469 mtx_unlock(&e->lock);
470 }
471 #endif
472