1 /*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include "opt_inet.h"
32
33 #include <sys/param.h>
34 #include <sys/types.h>
35 #include <sys/kernel.h>
36 #include <sys/queue.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/socket.h>
40 #include <sys/taskqueue.h>
41 #include <netinet/in.h>
42 #include <netinet/tcp.h>
43 #include <netinet/toecore.h>
44
45 #ifdef TCP_OFFLOAD
46 #include "cxgb_include.h"
47 #include "ulp/tom/cxgb_tom.h"
48 #include "ulp/tom/cxgb_l2t.h"
49 #include "ulp/tom/cxgb_toepcb.h"
50
51 MALLOC_DEFINE(M_CXGB, "cxgb", "Chelsio T3 Offload services");
52
53 /* Module ops */
54 static int t3_tom_mod_load(void);
55 static int t3_tom_mod_unload(void);
56 static int t3_tom_modevent(module_t, int, void *);
57
58 /* ULD ops and helpers */
59 static int t3_tom_activate(struct adapter *);
60 static int t3_tom_deactivate(struct adapter *);
61
62 static int alloc_tid_tabs(struct tid_info *, u_int, u_int, u_int, u_int, u_int);
63 static void free_tid_tabs(struct tid_info *);
64 static int write_smt_entry(struct adapter *, int);
65 static void free_tom_data(struct tom_data *);
66
67 static struct uld_info tom_uld_info = {
68 .uld_id = ULD_TOM,
69 .activate = t3_tom_activate,
70 .deactivate = t3_tom_deactivate,
71 };
72
73 struct toepcb *
toepcb_alloc(struct toedev * tod)74 toepcb_alloc(struct toedev *tod)
75 {
76 struct toepcb *toep;
77
78 toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT | M_ZERO);
79 if (toep == NULL)
80 return (NULL);
81
82 toep->tp_tod = tod;
83 toep->tp_wr_max = toep->tp_wr_avail = 15;
84 toep->tp_wr_unacked = 0;
85 toep->tp_delack_mode = 0;
86
87 return (toep);
88 }
89
90 void
toepcb_free(struct toepcb * toep)91 toepcb_free(struct toepcb *toep)
92 {
93 free(toep, M_CXGB);
94 }
95
96 static int
alloc_tid_tabs(struct tid_info * t,u_int ntids,u_int natids,u_int nstids,u_int atid_base,u_int stid_base)97 alloc_tid_tabs(struct tid_info *t, u_int ntids, u_int natids, u_int nstids,
98 u_int atid_base, u_int stid_base)
99 {
100 unsigned long size = ntids * sizeof(*t->tid_tab) +
101 natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab);
102
103 t->tid_tab = malloc(size, M_CXGB, M_NOWAIT | M_ZERO);
104 if (!t->tid_tab)
105 return (ENOMEM);
106
107 t->stid_tab = (union listen_entry *)&t->tid_tab[ntids];
108 t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids];
109 t->ntids = ntids;
110 t->nstids = nstids;
111 t->stid_base = stid_base;
112 t->sfree = NULL;
113 t->natids = natids;
114 t->atid_base = atid_base;
115 t->afree = NULL;
116 t->stids_in_use = t->atids_in_use = 0;
117 t->tids_in_use = 0;
118 mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF);
119 mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF);
120
121 /*
122 * Setup the free lists for stid_tab and atid_tab.
123 */
124 if (nstids) {
125 while (--nstids)
126 t->stid_tab[nstids - 1].next = &t->stid_tab[nstids];
127 t->sfree = t->stid_tab;
128 }
129 if (natids) {
130 while (--natids)
131 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
132 t->afree = t->atid_tab;
133 }
134 return (0);
135 }
136
137 static void
free_tid_tabs(struct tid_info * t)138 free_tid_tabs(struct tid_info *t)
139 {
140 if (mtx_initialized(&t->stid_lock))
141 mtx_destroy(&t->stid_lock);
142 if (mtx_initialized(&t->atid_lock))
143 mtx_destroy(&t->atid_lock);
144 free(t->tid_tab, M_CXGB);
145 }
146
147 static int
write_smt_entry(struct adapter * sc,int idx)148 write_smt_entry(struct adapter *sc, int idx)
149 {
150 struct port_info *pi = &sc->port[idx];
151 struct cpl_smt_write_req *req;
152 struct mbuf *m;
153
154 m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, req);
155 if (m == NULL) {
156 log(LOG_ERR, "%s: no mbuf, can't write SMT entry for %d\n",
157 __func__, idx);
158 return (ENOMEM);
159 }
160
161 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
162 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
163 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
164 req->iff = idx;
165 memset(req->src_mac1, 0, sizeof(req->src_mac1));
166 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
167
168 t3_offload_tx(sc, m);
169
170 return (0);
171 }
172
173 static void
free_tom_data(struct tom_data * td)174 free_tom_data(struct tom_data *td)
175 {
176 KASSERT(TAILQ_EMPTY(&td->toep_list),
177 ("%s: toep_list not empty", __func__));
178
179 if (td->listen_mask != 0)
180 hashdestroy(td->listen_hash, M_CXGB, td->listen_mask);
181
182 if (mtx_initialized(&td->toep_list_lock))
183 mtx_destroy(&td->toep_list_lock);
184 if (mtx_initialized(&td->lctx_hash_lock))
185 mtx_destroy(&td->lctx_hash_lock);
186 if (mtx_initialized(&td->tid_release_lock))
187 mtx_destroy(&td->tid_release_lock);
188 if (td->l2t)
189 t3_free_l2t(td->l2t);
190 free_tid_tabs(&td->tid_maps);
191 free(td, M_CXGB);
192 }
193
194 /*
195 * Ground control to Major TOM
196 * Commencing countdown, engines on
197 */
198 static int
t3_tom_activate(struct adapter * sc)199 t3_tom_activate(struct adapter *sc)
200 {
201 struct tom_data *td;
202 struct toedev *tod;
203 int i, rc = 0;
204 struct mc5_params *mc5 = &sc->params.mc5;
205 u_int ntids, natids, mtus;
206
207 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */
208
209 /* per-adapter softc for TOM */
210 td = malloc(sizeof(*td), M_CXGB, M_ZERO | M_NOWAIT);
211 if (td == NULL)
212 return (ENOMEM);
213
214 /* List of TOE PCBs and associated lock */
215 mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF);
216 TAILQ_INIT(&td->toep_list);
217
218 /* Listen context */
219 mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF);
220 td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGB,
221 &td->listen_mask, HASH_NOWAIT);
222
223 /* TID release task */
224 TASK_INIT(&td->tid_release_task, 0 , t3_process_tid_release_list, td);
225 mtx_init(&td->tid_release_lock, "tid release", NULL, MTX_DEF);
226
227 /* L2 table */
228 td->l2t = t3_init_l2t(L2T_SIZE);
229 if (td->l2t == NULL) {
230 rc = ENOMEM;
231 goto done;
232 }
233
234 /* TID tables */
235 ntids = t3_mc5_size(&sc->mc5) - mc5->nroutes - mc5->nfilters -
236 mc5->nservers;
237 natids = min(ntids / 2, 64 * 1024);
238 rc = alloc_tid_tabs(&td->tid_maps, ntids, natids, mc5->nservers,
239 0x100000 /* ATID_BASE */, ntids);
240 if (rc != 0)
241 goto done;
242
243 /* CPL handlers */
244 t3_init_listen_cpl_handlers(sc);
245 t3_init_l2t_cpl_handlers(sc);
246 t3_init_cpl_io(sc);
247
248 /* toedev ops */
249 tod = &td->tod;
250 init_toedev(tod);
251 tod->tod_softc = sc;
252 tod->tod_connect = t3_connect;
253 tod->tod_listen_start = t3_listen_start;
254 tod->tod_listen_stop = t3_listen_stop;
255 tod->tod_rcvd = t3_rcvd;
256 tod->tod_output = t3_tod_output;
257 tod->tod_send_rst = t3_send_rst;
258 tod->tod_send_fin = t3_send_fin;
259 tod->tod_pcb_detach = t3_pcb_detach;
260 tod->tod_l2_update = t3_l2_update;
261 tod->tod_syncache_added = t3_syncache_added;
262 tod->tod_syncache_removed = t3_syncache_removed;
263 tod->tod_syncache_respond = t3_syncache_respond;
264 tod->tod_offload_socket = t3_offload_socket;
265
266 /* port MTUs */
267 mtus = sc->port[0].ifp->if_mtu;
268 if (sc->params.nports > 1)
269 mtus |= sc->port[1].ifp->if_mtu << 16;
270 t3_write_reg(sc, A_TP_MTU_PORT_TABLE, mtus);
271 t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
272 sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff);
273
274 /* SMT entry for each port */
275 for_each_port(sc, i) {
276 write_smt_entry(sc, i);
277 TOEDEV(sc->port[i].ifp) = &td->tod;
278 }
279
280 /* Switch TP to offload mode */
281 t3_tp_set_offload_mode(sc, 1);
282
283 sc->tom_softc = td;
284 sc->flags |= TOM_INIT_DONE;
285 register_toedev(tod);
286
287 done:
288 if (rc != 0)
289 free_tom_data(td);
290
291 return (rc);
292 }
293
294 static int
t3_tom_deactivate(struct adapter * sc)295 t3_tom_deactivate(struct adapter *sc)
296 {
297 int rc = 0;
298 struct tom_data *td = sc->tom_softc;
299
300 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */
301
302 if (td == NULL)
303 return (0); /* XXX. KASSERT? */
304
305 if (sc->offload_map != 0)
306 return (EBUSY); /* at least one port has IFCAP_TOE enabled */
307
308 mtx_lock(&td->toep_list_lock);
309 if (!TAILQ_EMPTY(&td->toep_list))
310 rc = EBUSY;
311 mtx_unlock(&td->toep_list_lock);
312
313 mtx_lock(&td->lctx_hash_lock);
314 if (td->lctx_count > 0)
315 rc = EBUSY;
316 mtx_unlock(&td->lctx_hash_lock);
317
318 if (rc == 0) {
319 unregister_toedev(&td->tod);
320 t3_tp_set_offload_mode(sc, 0);
321 free_tom_data(td);
322 sc->tom_softc = NULL;
323 sc->flags &= ~TOM_INIT_DONE;
324 }
325
326 return (rc);
327 }
328
329 static int
t3_tom_mod_load(void)330 t3_tom_mod_load(void)
331 {
332 int rc;
333
334 rc = t3_register_uld(&tom_uld_info);
335 if (rc != 0)
336 t3_tom_mod_unload();
337
338 return (rc);
339 }
340
341 static void
tom_uninit(struct adapter * sc,void * arg __unused)342 tom_uninit(struct adapter *sc, void *arg __unused)
343 {
344 /* Try to free resources (works only if no port has IFCAP_TOE) */
345 ADAPTER_LOCK(sc);
346 if (sc->flags & TOM_INIT_DONE)
347 t3_deactivate_uld(sc, ULD_TOM);
348 ADAPTER_UNLOCK(sc);
349 }
350
351 static int
t3_tom_mod_unload(void)352 t3_tom_mod_unload(void)
353 {
354 t3_iterate(tom_uninit, NULL);
355
356 if (t3_unregister_uld(&tom_uld_info) == EBUSY)
357 return (EBUSY);
358
359 return (0);
360 }
361 #endif /* ifdef TCP_OFFLOAD */
362
363 static int
t3_tom_modevent(module_t mod,int cmd,void * arg)364 t3_tom_modevent(module_t mod, int cmd, void *arg)
365 {
366 int rc = 0;
367
368 #ifdef TCP_OFFLOAD
369 switch (cmd) {
370 case MOD_LOAD:
371 rc = t3_tom_mod_load();
372 break;
373
374 case MOD_UNLOAD:
375 rc = t3_tom_mod_unload();
376 break;
377
378 default:
379 rc = EINVAL;
380 }
381 #else
382 rc = EOPNOTSUPP;
383 #endif
384 return (rc);
385 }
386
387 static moduledata_t t3_tom_moddata= {
388 "t3_tom",
389 t3_tom_modevent,
390 0
391 };
392
393 MODULE_VERSION(t3_tom, 1);
394 MODULE_DEPEND(t3_tom, toecore, 1, 1, 1);
395 MODULE_DEPEND(t3_tom, cxgbc, 1, 1, 1);
396 DECLARE_MODULE(t3_tom, t3_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY);
397