1 /*-
2 * Copyright (c) 2012 The FreeBSD Foundation
3 * Copyright (c) 2015 Chelsio Communications, Inc.
4 * All rights reserved.
5 *
6 * This software was developed by Edward Tomasz Napierala under sponsorship
7 * from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 /*
33 * cxgbei implementation of iSCSI Common Layer kobj(9) interface.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_inet.h"
38 #include "opt_inet6.h"
39
40 #ifdef TCP_OFFLOAD
41 #include <sys/param.h>
42 #include <sys/bio.h>
43 #include <sys/capsicum.h>
44 #include <sys/condvar.h>
45 #include <sys/conf.h>
46 #include <sys/file.h>
47 #include <sys/kernel.h>
48 #include <sys/kthread.h>
49 #include <sys/ktr.h>
50 #include <sys/lock.h>
51 #include <sys/mbuf.h>
52 #include <sys/mutex.h>
53 #include <sys/module.h>
54 #include <sys/protosw.h>
55 #include <sys/socket.h>
56 #include <sys/socketvar.h>
57 #include <sys/sysctl.h>
58 #include <sys/systm.h>
59 #include <sys/sx.h>
60 #include <sys/uio.h>
61 #include <machine/bus.h>
62 #include <vm/vm.h>
63 #include <vm/vm_page.h>
64 #include <vm/pmap.h>
65 #include <netinet/in.h>
66 #include <netinet/in_pcb.h>
67 #include <netinet/tcp.h>
68 #include <netinet/tcp_var.h>
69 #include <netinet/toecore.h>
70
71 #include <dev/iscsi/icl.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <icl_conn_if.h>
74
75 #include <cam/scsi/scsi_all.h>
76 #include <cam/scsi/scsi_da.h>
77 #include <cam/ctl/ctl_io.h>
78 #include <cam/ctl/ctl.h>
79 #include <cam/ctl/ctl_backend.h>
80 #include <cam/ctl/ctl_error.h>
81 #include <cam/ctl/ctl_frontend.h>
82 #include <cam/ctl/ctl_debug.h>
83 #include <cam/ctl/ctl_ha.h>
84 #include <cam/ctl/ctl_ioctl.h>
85
86 #include <cam/cam.h>
87 #include <cam/cam_ccb.h>
88 #include <cam/cam_xpt.h>
89 #include <cam/cam_debug.h>
90 #include <cam/cam_sim.h>
91 #include <cam/cam_xpt_sim.h>
92 #include <cam/cam_xpt_periph.h>
93 #include <cam/cam_periph.h>
94 #include <cam/cam_compat.h>
95 #include <cam/scsi/scsi_message.h>
96
97 #include "common/common.h"
98 #include "common/t4_regs.h"
99 #include "common/t4_tcb.h"
100 #include "tom/t4_tom.h"
101 #include "cxgbei.h"
102
103 /*
104 * Use the page pod tag for the TT hash.
105 */
106 #define TT_HASH(icc, tt) (G_PPOD_TAG(tt) & (icc)->cmp_hash_mask)
107
108 struct cxgbei_ddp_state {
109 struct ppod_reservation prsv;
110 struct cxgbei_cmp cmp;
111 };
112
113 static MALLOC_DEFINE(M_CXGBEI, "cxgbei", "cxgbei(4)");
114
115 SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
116 "Chelsio iSCSI offload");
117 static int first_burst_length = 8192;
118 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN,
119 &first_burst_length, 0, "First burst length");
120 static int max_burst_length = 2 * 1024 * 1024;
121 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN,
122 &max_burst_length, 0, "Maximum burst length");
123 static int sendspace = 1048576;
124 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN,
125 &sendspace, 0, "Default send socket buffer size");
126 static int recvspace = 1048576;
127 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN,
128 &recvspace, 0, "Default receive socket buffer size");
129
130 static volatile u_int icl_cxgbei_ncons;
131
132 static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu;
133 static icl_conn_pdu_data_segment_length_t
134 icl_cxgbei_conn_pdu_data_segment_length;
135 static icl_conn_pdu_append_bio_t icl_cxgbei_conn_pdu_append_bio;
136 static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data;
137 static icl_conn_pdu_get_bio_t icl_cxgbei_conn_pdu_get_bio;
138 static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data;
139 static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue;
140 static icl_conn_pdu_queue_cb_t icl_cxgbei_conn_pdu_queue_cb;
141 static icl_conn_handoff_t icl_cxgbei_conn_handoff;
142 static icl_conn_free_t icl_cxgbei_conn_free;
143 static icl_conn_close_t icl_cxgbei_conn_close;
144 static icl_conn_task_setup_t icl_cxgbei_conn_task_setup;
145 static icl_conn_task_done_t icl_cxgbei_conn_task_done;
146 static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup;
147 static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done;
148
149 static kobj_method_t icl_cxgbei_methods[] = {
150 KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu),
151 KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free),
152 KOBJMETHOD(icl_conn_pdu_data_segment_length,
153 icl_cxgbei_conn_pdu_data_segment_length),
154 KOBJMETHOD(icl_conn_pdu_append_bio, icl_cxgbei_conn_pdu_append_bio),
155 KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data),
156 KOBJMETHOD(icl_conn_pdu_get_bio, icl_cxgbei_conn_pdu_get_bio),
157 KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data),
158 KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue),
159 KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb),
160 KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff),
161 KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free),
162 KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close),
163 KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup),
164 KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done),
165 KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup),
166 KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done),
167 { 0, 0 }
168 };
169
170 DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn));
171
172 void
icl_cxgbei_conn_pdu_free(struct icl_conn * ic,struct icl_pdu * ip)173 icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
174 {
175 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
176
177 KASSERT(icp->ref_cnt != 0, ("freeing deleted PDU"));
178 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
179 MPASS(ic == ip->ip_conn);
180
181 m_freem(ip->ip_ahs_mbuf);
182 m_freem(ip->ip_data_mbuf);
183 m_freem(ip->ip_bhs_mbuf);
184
185 KASSERT(ic != NULL || icp->ref_cnt == 1,
186 ("orphaned PDU has oustanding references"));
187
188 if (atomic_fetchadd_int(&icp->ref_cnt, -1) != 1)
189 return;
190
191 free(icp, M_CXGBEI);
192 #ifdef DIAGNOSTIC
193 if (__predict_true(ic != NULL))
194 refcount_release(&ic->ic_outstanding_pdus);
195 #endif
196 }
197
198 static void
icl_cxgbei_pdu_call_cb(struct icl_pdu * ip)199 icl_cxgbei_pdu_call_cb(struct icl_pdu *ip)
200 {
201 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
202
203 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
204
205 if (icp->cb != NULL)
206 icp->cb(ip, icp->error);
207 #ifdef DIAGNOSTIC
208 if (__predict_true(ip->ip_conn != NULL))
209 refcount_release(&ip->ip_conn->ic_outstanding_pdus);
210 #endif
211 free(icp, M_CXGBEI);
212 }
213
214 static void
icl_cxgbei_pdu_done(struct icl_pdu * ip,int error)215 icl_cxgbei_pdu_done(struct icl_pdu *ip, int error)
216 {
217 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
218
219 if (error != 0)
220 icp->error = error;
221
222 m_freem(ip->ip_ahs_mbuf);
223 ip->ip_ahs_mbuf = NULL;
224 m_freem(ip->ip_data_mbuf);
225 ip->ip_data_mbuf = NULL;
226 m_freem(ip->ip_bhs_mbuf);
227 ip->ip_bhs_mbuf = NULL;
228
229 /*
230 * All other references to this PDU should have been dropped
231 * by the m_freem() of ip_data_mbuf.
232 */
233 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
234 icl_cxgbei_pdu_call_cb(ip);
235 else
236 __assert_unreachable();
237 }
238
239 static void
icl_cxgbei_mbuf_done(struct mbuf * mb)240 icl_cxgbei_mbuf_done(struct mbuf *mb)
241 {
242
243 struct icl_cxgbei_pdu *icp = (struct icl_cxgbei_pdu *)mb->m_ext.ext_arg1;
244
245 /*
246 * NB: mb_free_mext() might leave ref_cnt as 1 without
247 * decrementing it if it hits the fast path in the ref_cnt
248 * check.
249 */
250 icl_cxgbei_pdu_call_cb(&icp->ip);
251 }
252
253 struct icl_pdu *
icl_cxgbei_new_pdu(int flags)254 icl_cxgbei_new_pdu(int flags)
255 {
256 struct icl_cxgbei_pdu *icp;
257 struct icl_pdu *ip;
258 struct mbuf *m;
259
260 icp = malloc(sizeof(*icp), M_CXGBEI, flags | M_ZERO);
261 if (__predict_false(icp == NULL))
262 return (NULL);
263
264 icp->icp_signature = CXGBEI_PDU_SIGNATURE;
265 icp->ref_cnt = 1;
266 ip = &icp->ip;
267
268 m = m_gethdr(flags, MT_DATA);
269 if (__predict_false(m == NULL)) {
270 free(icp, M_CXGBEI);
271 return (NULL);
272 }
273
274 ip->ip_bhs_mbuf = m;
275 ip->ip_bhs = mtod(m, struct iscsi_bhs *);
276 memset(ip->ip_bhs, 0, sizeof(*ip->ip_bhs));
277 m->m_len = sizeof(struct iscsi_bhs);
278 m->m_pkthdr.len = m->m_len;
279
280 return (ip);
281 }
282
283 void
icl_cxgbei_new_pdu_set_conn(struct icl_pdu * ip,struct icl_conn * ic)284 icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic)
285 {
286
287 ip->ip_conn = ic;
288 #ifdef DIAGNOSTIC
289 refcount_acquire(&ic->ic_outstanding_pdus);
290 #endif
291 }
292
293 /*
294 * Allocate icl_pdu with empty BHS to fill up by the caller.
295 */
296 static struct icl_pdu *
icl_cxgbei_conn_new_pdu(struct icl_conn * ic,int flags)297 icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags)
298 {
299 struct icl_pdu *ip;
300
301 ip = icl_cxgbei_new_pdu(flags);
302 if (__predict_false(ip == NULL))
303 return (NULL);
304 icl_cxgbei_new_pdu_set_conn(ip, ic);
305
306 return (ip);
307 }
308
309 static size_t
icl_pdu_data_segment_length(const struct icl_pdu * request)310 icl_pdu_data_segment_length(const struct icl_pdu *request)
311 {
312 uint32_t len = 0;
313
314 len += request->ip_bhs->bhs_data_segment_len[0];
315 len <<= 8;
316 len += request->ip_bhs->bhs_data_segment_len[1];
317 len <<= 8;
318 len += request->ip_bhs->bhs_data_segment_len[2];
319
320 return (len);
321 }
322
323 size_t
icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn * ic,const struct icl_pdu * request)324 icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic,
325 const struct icl_pdu *request)
326 {
327
328 return (icl_pdu_data_segment_length(request));
329 }
330
331 static struct mbuf *
finalize_pdu(struct icl_cxgbei_conn * icc,struct icl_cxgbei_pdu * icp)332 finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)
333 {
334 struct icl_pdu *ip = &icp->ip;
335 uint8_t ulp_submode, padding;
336 struct mbuf *m, *last;
337 struct iscsi_bhs *bhs;
338 int data_len;
339
340 /*
341 * Fix up the data segment mbuf first.
342 */
343 m = ip->ip_data_mbuf;
344 ulp_submode = icc->ulp_submode;
345 if (m != NULL) {
346 last = m_last(m);
347
348 /*
349 * Round up the data segment to a 4B boundary. Pad with 0 if
350 * necessary. There will definitely be room in the mbuf.
351 */
352 padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len;
353 if (padding != 0) {
354 MPASS(padding <= M_TRAILINGSPACE(last));
355 bzero(mtod(last, uint8_t *) + last->m_len, padding);
356 last->m_len += padding;
357 }
358 } else {
359 MPASS(ip->ip_data_len == 0);
360 ulp_submode &= ~ULP_CRC_DATA;
361 padding = 0;
362 }
363
364 /*
365 * Now the header mbuf that has the BHS.
366 */
367 m = ip->ip_bhs_mbuf;
368 MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs));
369 MPASS(m->m_len == sizeof(struct iscsi_bhs));
370
371 bhs = ip->ip_bhs;
372 data_len = ip->ip_data_len;
373 if (data_len > icc->ic.ic_max_send_data_segment_length) {
374 struct iscsi_bhs_data_in *bhsdi;
375 int flags;
376
377 KASSERT(padding == 0, ("%s: ISO with padding %d for icp %p",
378 __func__, padding, icp));
379 switch (bhs->bhs_opcode) {
380 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
381 flags = 1;
382 break;
383 case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
384 flags = 2;
385 break;
386 default:
387 panic("invalid opcode %#x for ISO", bhs->bhs_opcode);
388 }
389 data_len = icc->ic.ic_max_send_data_segment_length;
390 bhsdi = (struct iscsi_bhs_data_in *)bhs;
391 if (bhsdi->bhsdi_flags & BHSDI_FLAGS_F) {
392 /*
393 * Firmware will set F on the final PDU in the
394 * burst.
395 */
396 flags |= CXGBE_ISO_F;
397 bhsdi->bhsdi_flags &= ~BHSDI_FLAGS_F;
398 }
399 set_mbuf_iscsi_iso(m, true);
400 set_mbuf_iscsi_iso_flags(m, flags);
401 set_mbuf_iscsi_iso_mss(m, data_len);
402 }
403
404 bhs->bhs_data_segment_len[2] = data_len;
405 bhs->bhs_data_segment_len[1] = data_len >> 8;
406 bhs->bhs_data_segment_len[0] = data_len >> 16;
407
408 /*
409 * Extract mbuf chain from PDU.
410 */
411 m->m_pkthdr.len += ip->ip_data_len + padding;
412 m->m_next = ip->ip_data_mbuf;
413 set_mbuf_ulp_submode(m, ulp_submode);
414 ip->ip_bhs_mbuf = NULL;
415 ip->ip_data_mbuf = NULL;
416 ip->ip_bhs = NULL;
417
418 /*
419 * Drop PDU reference on icp. Additional references might
420 * still be held by zero-copy PDU buffers (ICL_NOCOPY).
421 */
422 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
423 icl_cxgbei_pdu_call_cb(ip);
424
425 return (m);
426 }
427
428 static void
icl_cxgbei_tx_main(void * arg)429 icl_cxgbei_tx_main(void *arg)
430 {
431 struct epoch_tracker et;
432 struct icl_cxgbei_conn *icc = arg;
433 struct icl_conn *ic = &icc->ic;
434 struct toepcb *toep = icc->toep;
435 struct socket *so = ic->ic_socket;
436 struct inpcb *inp = sotoinpcb(so);
437 struct icl_pdu *ip;
438 struct mbuf *m;
439 struct mbufq mq;
440 STAILQ_HEAD(, icl_pdu) tx_pdus = STAILQ_HEAD_INITIALIZER(tx_pdus);
441
442 mbufq_init(&mq, INT_MAX);
443
444 ICL_CONN_LOCK(ic);
445 while (__predict_true(!ic->ic_disconnecting)) {
446 while (STAILQ_EMPTY(&icc->sent_pdus)) {
447 icc->tx_active = false;
448 mtx_sleep(&icc->tx_active, ic->ic_lock, 0, "-", 0);
449 if (__predict_false(ic->ic_disconnecting))
450 goto out;
451 MPASS(icc->tx_active);
452 }
453
454 STAILQ_SWAP(&icc->sent_pdus, &tx_pdus, icl_pdu);
455 ICL_CONN_UNLOCK(ic);
456
457 while ((ip = STAILQ_FIRST(&tx_pdus)) != NULL) {
458 STAILQ_REMOVE_HEAD(&tx_pdus, ip_next);
459
460 m = finalize_pdu(icc, ip_to_icp(ip));
461 M_ASSERTPKTHDR(m);
462 MPASS((m->m_pkthdr.len & 3) == 0);
463
464 mbufq_enqueue(&mq, m);
465 }
466
467 ICL_CONN_LOCK(ic);
468 if (__predict_false(ic->ic_disconnecting) ||
469 __predict_false(ic->ic_socket == NULL)) {
470 mbufq_drain(&mq);
471 break;
472 }
473
474 CURVNET_SET(toep->vnet);
475 NET_EPOCH_ENTER(et);
476 INP_WLOCK(inp);
477
478 ICL_CONN_UNLOCK(ic);
479 if (__predict_false(inp->inp_flags & (INP_DROPPED |
480 INP_TIMEWAIT)) ||
481 __predict_false((toep->flags & TPF_ATTACHED) == 0)) {
482 mbufq_drain(&mq);
483 } else {
484 mbufq_concat(&toep->ulp_pduq, &mq);
485 t4_push_pdus(icc->sc, toep, 0);
486 }
487 INP_WUNLOCK(inp);
488 NET_EPOCH_EXIT(et);
489 CURVNET_RESTORE();
490
491 ICL_CONN_LOCK(ic);
492 }
493 out:
494 ICL_CONN_UNLOCK(ic);
495
496 kthread_exit();
497 }
498
499 static void
icl_cxgbei_rx_main(void * arg)500 icl_cxgbei_rx_main(void *arg)
501 {
502 struct icl_cxgbei_conn *icc = arg;
503 struct icl_conn *ic = &icc->ic;
504 struct icl_pdu *ip;
505 struct sockbuf *sb;
506 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus);
507 bool cantrcvmore;
508
509 sb = &ic->ic_socket->so_rcv;
510 SOCKBUF_LOCK(sb);
511 while (__predict_true(!ic->ic_disconnecting)) {
512 while (STAILQ_EMPTY(&icc->rcvd_pdus)) {
513 icc->rx_active = false;
514 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0);
515 if (__predict_false(ic->ic_disconnecting))
516 goto out;
517 MPASS(icc->rx_active);
518 }
519
520 if (__predict_false(sbused(sb)) != 0) {
521 /*
522 * PDUs were received before the tid
523 * transitioned to ULP mode. Convert
524 * them to icl_cxgbei_pdus and insert
525 * them into the head of rcvd_pdus.
526 */
527 parse_pdus(icc, sb);
528 }
529 cantrcvmore = (sb->sb_state & SBS_CANTRCVMORE) != 0;
530 MPASS(STAILQ_EMPTY(&rx_pdus));
531 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu);
532 SOCKBUF_UNLOCK(sb);
533
534 /* Hand over PDUs to ICL. */
535 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) {
536 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next);
537 if (cantrcvmore)
538 icl_cxgbei_pdu_done(ip, ENOTCONN);
539 else
540 ic->ic_receive(ip);
541 }
542
543 SOCKBUF_LOCK(sb);
544 }
545 out:
546 /*
547 * Since ic_disconnecting is set before the SOCKBUF_MTX is
548 * locked in icl_cxgbei_conn_close, the loop above can exit
549 * before icl_cxgbei_conn_close can lock SOCKBUF_MTX and block
550 * waiting for the thread exit.
551 */
552 while (!icc->rx_exiting)
553 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0);
554 SOCKBUF_UNLOCK(sb);
555
556 kthread_exit();
557 }
558
559 static void
cxgbei_free_mext_pg(struct mbuf * m)560 cxgbei_free_mext_pg(struct mbuf *m)
561 {
562 struct icl_cxgbei_pdu *icp;
563
564 M_ASSERTEXTPG(m);
565
566 /*
567 * Nothing to do for the pages; they are owned by the PDU /
568 * I/O request.
569 */
570
571 /* Drop reference on the PDU. */
572 icp = m->m_ext.ext_arg1;
573 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
574 icl_cxgbei_pdu_call_cb(&icp->ip);
575 }
576
577 static struct mbuf *
cxgbei_getm(size_t len,int flags)578 cxgbei_getm(size_t len, int flags)
579 {
580 struct mbuf *m, *m0, *m_tail;
581
582 m_tail = m0 = NULL;
583
584 /* Allocate as jumbo mbufs of size MJUM16BYTES. */
585 while (len >= MJUM16BYTES) {
586 m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
587 if (__predict_false(m == NULL)) {
588 if ((flags & M_WAITOK) != 0) {
589 /* Fall back to non-jumbo mbufs. */
590 break;
591 }
592 return (NULL);
593 }
594 if (m0 == NULL) {
595 m0 = m_tail = m;
596 } else {
597 m_tail->m_next = m;
598 m_tail = m;
599 }
600 len -= MJUM16BYTES;
601 }
602
603 /* Allocate mbuf chain for the remaining data. */
604 if (len != 0) {
605 m = m_getm2(NULL, len, flags, MT_DATA, 0);
606 if (__predict_false(m == NULL)) {
607 m_freem(m0);
608 return (NULL);
609 }
610 if (m0 == NULL)
611 m0 = m;
612 else
613 m_tail->m_next = m;
614 }
615
616 return (m0);
617 }
618
619 int
icl_cxgbei_conn_pdu_append_bio(struct icl_conn * ic,struct icl_pdu * ip,struct bio * bp,size_t offset,size_t len,int flags)620 icl_cxgbei_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *ip,
621 struct bio *bp, size_t offset, size_t len, int flags)
622 {
623 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
624 struct mbuf *m, *m_tail;
625 vm_offset_t vaddr;
626 size_t page_offset, todo, mtodo;
627 boolean_t mapped;
628 int i;
629
630 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
631 MPASS(ic == ip->ip_conn);
632 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len));
633
634 m_tail = ip->ip_data_mbuf;
635 if (m_tail != NULL)
636 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
637 ;
638
639 MPASS(bp->bio_flags & BIO_UNMAPPED);
640 if (offset < PAGE_SIZE - bp->bio_ma_offset) {
641 page_offset = bp->bio_ma_offset + offset;
642 i = 0;
643 } else {
644 offset -= PAGE_SIZE - bp->bio_ma_offset;
645 for (i = 1; offset >= PAGE_SIZE; i++)
646 offset -= PAGE_SIZE;
647 page_offset = offset;
648 }
649
650 if (flags & ICL_NOCOPY) {
651 m = NULL;
652 while (len > 0) {
653 if (m == NULL) {
654 m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY,
655 cxgbei_free_mext_pg);
656 if (__predict_false(m == NULL))
657 return (ENOMEM);
658 atomic_add_int(&icp->ref_cnt, 1);
659 m->m_ext.ext_arg1 = icp;
660 m->m_epg_1st_off = page_offset;
661 }
662
663 todo = MIN(len, PAGE_SIZE - page_offset);
664
665 m->m_epg_pa[m->m_epg_npgs] =
666 VM_PAGE_TO_PHYS(bp->bio_ma[i]);
667 m->m_epg_npgs++;
668 m->m_epg_last_len = todo;
669 m->m_len += todo;
670 m->m_ext.ext_size += PAGE_SIZE;
671 MBUF_EXT_PGS_ASSERT_SANITY(m);
672
673 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) {
674 if (m_tail != NULL)
675 m_tail->m_next = m;
676 else
677 ip->ip_data_mbuf = m;
678 m_tail = m;
679 ip->ip_data_len += m->m_len;
680 m = NULL;
681 }
682
683 page_offset = 0;
684 len -= todo;
685 i++;
686 }
687
688 if (m != NULL) {
689 if (m_tail != NULL)
690 m_tail->m_next = m;
691 else
692 ip->ip_data_mbuf = m;
693 ip->ip_data_len += m->m_len;
694 }
695 return (0);
696 }
697
698 m = cxgbei_getm(len, flags);
699 if (__predict_false(m == NULL))
700 return (ENOMEM);
701
702 if (ip->ip_data_mbuf == NULL) {
703 ip->ip_data_mbuf = m;
704 ip->ip_data_len = len;
705 } else {
706 m_tail->m_next = m;
707 ip->ip_data_len += len;
708 }
709
710 while (len > 0) {
711 todo = MIN(len, PAGE_SIZE - page_offset);
712
713 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
714 FALSE);
715
716 do {
717 mtodo = min(todo, M_SIZE(m) - m->m_len);
718 memcpy(mtod(m, char *) + m->m_len, (char *)vaddr +
719 page_offset, mtodo);
720 m->m_len += mtodo;
721 if (m->m_len == M_SIZE(m))
722 m = m->m_next;
723 page_offset += mtodo;
724 todo -= mtodo;
725 } while (todo > 0);
726
727 if (__predict_false(mapped))
728 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
729 FALSE);
730
731 page_offset = 0;
732 len -= todo;
733 i++;
734 }
735
736 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
737 ic->ic_hw_isomax));
738
739 return (0);
740 }
741
742 int
icl_cxgbei_conn_pdu_append_data(struct icl_conn * ic,struct icl_pdu * ip,const void * addr,size_t len,int flags)743 icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip,
744 const void *addr, size_t len, int flags)
745 {
746 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
747 struct mbuf *m, *m_tail;
748 const char *src;
749
750 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
751 MPASS(ic == ip->ip_conn);
752 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len));
753
754 m_tail = ip->ip_data_mbuf;
755 if (m_tail != NULL)
756 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
757 ;
758
759 if (flags & ICL_NOCOPY) {
760 m = m_get(flags & ~ICL_NOCOPY, MT_DATA);
761 if (m == NULL) {
762 ICL_WARN("failed to allocate mbuf");
763 return (ENOMEM);
764 }
765
766 m->m_flags |= M_RDONLY;
767 m_extaddref(m, __DECONST(char *, addr), len, &icp->ref_cnt,
768 icl_cxgbei_mbuf_done, icp, NULL);
769 m->m_len = len;
770 if (ip->ip_data_mbuf == NULL) {
771 ip->ip_data_mbuf = m;
772 ip->ip_data_len = len;
773 } else {
774 m_tail->m_next = m;
775 m_tail = m_tail->m_next;
776 ip->ip_data_len += len;
777 }
778
779 return (0);
780 }
781
782 m = cxgbei_getm(len, flags);
783 if (__predict_false(m == NULL))
784 return (ENOMEM);
785
786 if (ip->ip_data_mbuf == NULL) {
787 ip->ip_data_mbuf = m;
788 ip->ip_data_len = len;
789 } else {
790 m_tail->m_next = m;
791 ip->ip_data_len += len;
792 }
793 src = (const char *)addr;
794 for (; m != NULL; m = m->m_next) {
795 m->m_len = min(len, M_SIZE(m));
796 memcpy(mtod(m, void *), src, m->m_len);
797 src += m->m_len;
798 len -= m->m_len;
799 }
800 MPASS(len == 0);
801
802 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
803 ic->ic_hw_isomax));
804
805 return (0);
806 }
807
808 void
icl_cxgbei_conn_pdu_get_bio(struct icl_conn * ic,struct icl_pdu * ip,size_t pdu_off,struct bio * bp,size_t bio_off,size_t len)809 icl_cxgbei_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip,
810 size_t pdu_off, struct bio *bp, size_t bio_off, size_t len)
811 {
812 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
813 vm_offset_t vaddr;
814 size_t page_offset, todo;
815 boolean_t mapped;
816 int i;
817
818 if (icp->icp_flags & ICPF_RX_DDP)
819 return; /* data is DDP'ed, no need to copy */
820
821 MPASS(bp->bio_flags & BIO_UNMAPPED);
822 if (bio_off < PAGE_SIZE - bp->bio_ma_offset) {
823 page_offset = bp->bio_ma_offset + bio_off;
824 i = 0;
825 } else {
826 bio_off -= PAGE_SIZE - bp->bio_ma_offset;
827 for (i = 1; bio_off >= PAGE_SIZE; i++)
828 bio_off -= PAGE_SIZE;
829 page_offset = bio_off;
830 }
831
832 while (len > 0) {
833 todo = MIN(len, PAGE_SIZE - page_offset);
834
835 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
836 FALSE);
837 m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr +
838 page_offset);
839 if (__predict_false(mapped))
840 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
841 FALSE);
842
843 page_offset = 0;
844 pdu_off += todo;
845 len -= todo;
846 i++;
847 }
848 }
849
850 void
icl_cxgbei_conn_pdu_get_data(struct icl_conn * ic,struct icl_pdu * ip,size_t off,void * addr,size_t len)851 icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
852 size_t off, void *addr, size_t len)
853 {
854 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
855
856 if (icp->icp_flags & ICPF_RX_DDP)
857 return; /* data is DDP'ed, no need to copy */
858 m_copydata(ip->ip_data_mbuf, off, len, addr);
859 }
860
861 void
icl_cxgbei_conn_pdu_queue(struct icl_conn * ic,struct icl_pdu * ip)862 icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
863 {
864 icl_cxgbei_conn_pdu_queue_cb(ic, ip, NULL);
865 }
866
867 void
icl_cxgbei_conn_pdu_queue_cb(struct icl_conn * ic,struct icl_pdu * ip,icl_pdu_cb cb)868 icl_cxgbei_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip,
869 icl_pdu_cb cb)
870 {
871 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
872 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
873 struct socket *so = ic->ic_socket;
874
875 MPASS(ic == ip->ip_conn);
876 MPASS(ip->ip_bhs_mbuf != NULL);
877 /* The kernel doesn't generate PDUs with AHS. */
878 MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0);
879
880 ICL_CONN_LOCK_ASSERT(ic);
881
882 icp->cb = cb;
883
884 /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */
885 if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) {
886 icl_cxgbei_pdu_done(ip, ENOTCONN);
887 return;
888 }
889
890 STAILQ_INSERT_TAIL(&icc->sent_pdus, ip, ip_next);
891 if (!icc->tx_active) {
892 icc->tx_active = true;
893 wakeup(&icc->tx_active);
894 }
895 }
896
897 static struct icl_conn *
icl_cxgbei_new_conn(const char * name,struct mtx * lock)898 icl_cxgbei_new_conn(const char *name, struct mtx *lock)
899 {
900 struct icl_cxgbei_conn *icc;
901 struct icl_conn *ic;
902
903 refcount_acquire(&icl_cxgbei_ncons);
904
905 icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE,
906 M_WAITOK | M_ZERO);
907 icc->icc_signature = CXGBEI_CONN_SIGNATURE;
908 STAILQ_INIT(&icc->rcvd_pdus);
909 STAILQ_INIT(&icc->sent_pdus);
910
911 icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask);
912 mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF);
913
914 ic = &icc->ic;
915 ic->ic_lock = lock;
916
917 #ifdef DIAGNOSTIC
918 refcount_init(&ic->ic_outstanding_pdus, 0);
919 #endif
920 ic->ic_name = name;
921 ic->ic_offload = "cxgbei";
922 ic->ic_unmapped = true;
923
924 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
925
926 return (ic);
927 }
928
929 void
icl_cxgbei_conn_free(struct icl_conn * ic)930 icl_cxgbei_conn_free(struct icl_conn *ic)
931 {
932 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
933
934 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
935
936 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
937
938 mtx_destroy(&icc->cmp_lock);
939 hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask);
940 kobj_delete((struct kobj *)icc, M_CXGBE);
941 refcount_release(&icl_cxgbei_ncons);
942 }
943
944 static int
icl_cxgbei_setsockopt(struct icl_conn * ic,struct socket * so,int sspace,int rspace)945 icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace,
946 int rspace)
947 {
948 struct sockopt opt;
949 int error, one = 1, ss, rs;
950
951 ss = max(sendspace, sspace);
952 rs = max(recvspace, rspace);
953
954 error = soreserve(so, ss, rs);
955 if (error != 0)
956 return (error);
957 SOCKBUF_LOCK(&so->so_snd);
958 so->so_snd.sb_flags |= SB_AUTOSIZE;
959 SOCKBUF_UNLOCK(&so->so_snd);
960 SOCKBUF_LOCK(&so->so_rcv);
961 so->so_rcv.sb_flags |= SB_AUTOSIZE;
962 SOCKBUF_UNLOCK(&so->so_rcv);
963
964 /*
965 * Disable Nagle.
966 */
967 bzero(&opt, sizeof(opt));
968 opt.sopt_dir = SOPT_SET;
969 opt.sopt_level = IPPROTO_TCP;
970 opt.sopt_name = TCP_NODELAY;
971 opt.sopt_val = &one;
972 opt.sopt_valsize = sizeof(one);
973 error = sosetopt(so, &opt);
974 if (error != 0)
975 return (error);
976
977 return (0);
978 }
979
980 /*
981 * Request/response structure used to find out the adapter offloading a socket.
982 */
983 struct find_ofld_adapter_rr {
984 struct socket *so;
985 struct adapter *sc; /* result */
986 };
987
988 static void
find_offload_adapter(struct adapter * sc,void * arg)989 find_offload_adapter(struct adapter *sc, void *arg)
990 {
991 struct find_ofld_adapter_rr *fa = arg;
992 struct socket *so = fa->so;
993 struct tom_data *td = sc->tom_softc;
994 struct tcpcb *tp;
995 struct inpcb *inp;
996
997 /* Non-TCP were filtered out earlier. */
998 MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
999
1000 if (fa->sc != NULL)
1001 return; /* Found already. */
1002
1003 if (td == NULL)
1004 return; /* TOE not enabled on this adapter. */
1005
1006 inp = sotoinpcb(so);
1007 INP_WLOCK(inp);
1008 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1009 tp = intotcpcb(inp);
1010 if (tp->t_flags & TF_TOE && tp->tod == &td->tod)
1011 fa->sc = sc; /* Found. */
1012 }
1013 INP_WUNLOCK(inp);
1014 }
1015
1016 static bool
is_memfree(struct adapter * sc)1017 is_memfree(struct adapter *sc)
1018 {
1019 uint32_t em;
1020
1021 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1022 if ((em & F_EXT_MEM_ENABLE) != 0)
1023 return (false);
1024 if (is_t5(sc) && (em & F_EXT_MEM1_ENABLE) != 0)
1025 return (false);
1026 return (true);
1027 }
1028
1029 /* XXXNP: move this to t4_tom. */
1030 static void
send_iscsi_flowc_wr(struct adapter * sc,struct toepcb * toep,int maxlen)1031 send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen)
1032 {
1033 struct wrqe *wr;
1034 struct fw_flowc_wr *flowc;
1035 const u_int nparams = 1;
1036 u_int flowclen;
1037 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
1038
1039 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
1040
1041 wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq);
1042 if (wr == NULL) {
1043 /* XXX */
1044 panic("%s: allocation failure.", __func__);
1045 }
1046 flowc = wrtod(wr);
1047 memset(flowc, 0, wr->wr_len);
1048
1049 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
1050 V_FW_FLOWC_WR_NPARAMS(nparams));
1051 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
1052 V_FW_WR_FLOWID(toep->tid));
1053
1054 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
1055 flowc->mnemval[0].val = htobe32(maxlen);
1056
1057 txsd->tx_credits = howmany(flowclen, 16);
1058 txsd->plen = 0;
1059 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
1060 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
1061 toep->tx_credits -= txsd->tx_credits;
1062 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
1063 toep->txsd_pidx = 0;
1064 toep->txsd_avail--;
1065
1066 t4_wrq_tx(sc, wr);
1067 }
1068
1069 static void
set_ulp_mode_iscsi(struct adapter * sc,struct toepcb * toep,u_int ulp_submode)1070 set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, u_int ulp_submode)
1071 {
1072 uint64_t val;
1073
1074 CTR3(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, submode=%#x",
1075 __func__, toep->tid, ulp_submode);
1076
1077 val = V_TCB_ULP_TYPE(ULP_MODE_ISCSI) | V_TCB_ULP_RAW(ulp_submode);
1078 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE,
1079 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val,
1080 0, 0);
1081
1082 val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL);
1083 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, val, val, 0, 0);
1084 }
1085
1086 /*
1087 * XXXNP: Who is responsible for cleaning up the socket if this returns with an
1088 * error? Review all error paths.
1089 *
1090 * XXXNP: What happens to the socket's fd reference if the operation is
1091 * successful, and how does that affect the socket's life cycle?
1092 */
1093 int
icl_cxgbei_conn_handoff(struct icl_conn * ic,int fd)1094 icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
1095 {
1096 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1097 struct find_ofld_adapter_rr fa;
1098 struct file *fp;
1099 struct socket *so;
1100 struct inpcb *inp;
1101 struct tcpcb *tp;
1102 struct toepcb *toep;
1103 cap_rights_t rights;
1104 u_int max_iso_payload, max_rx_pdu_len, max_tx_pdu_len;
1105 int error, max_iso_pdus;
1106
1107 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
1108 ICL_CONN_LOCK_ASSERT_NOT(ic);
1109
1110 /*
1111 * Steal the socket from userland.
1112 */
1113 error = fget(curthread, fd,
1114 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
1115 if (error != 0)
1116 return (error);
1117 if (fp->f_type != DTYPE_SOCKET) {
1118 fdrop(fp, curthread);
1119 return (EINVAL);
1120 }
1121 so = fp->f_data;
1122 if (so->so_type != SOCK_STREAM ||
1123 so->so_proto->pr_protocol != IPPROTO_TCP) {
1124 fdrop(fp, curthread);
1125 return (EINVAL);
1126 }
1127
1128 ICL_CONN_LOCK(ic);
1129 if (ic->ic_socket != NULL) {
1130 ICL_CONN_UNLOCK(ic);
1131 fdrop(fp, curthread);
1132 return (EBUSY);
1133 }
1134 ic->ic_disconnecting = false;
1135 ic->ic_socket = so;
1136 fp->f_ops = &badfileops;
1137 fp->f_data = NULL;
1138 fdrop(fp, curthread);
1139 ICL_CONN_UNLOCK(ic);
1140
1141 /* Find the adapter offloading this socket. */
1142 fa.sc = NULL;
1143 fa.so = so;
1144 t4_iterate(find_offload_adapter, &fa);
1145 if (fa.sc == NULL) {
1146 error = EINVAL;
1147 goto out;
1148 }
1149 icc->sc = fa.sc;
1150
1151 max_rx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_recv_data_segment_length;
1152 max_tx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_send_data_segment_length;
1153 if (ic->ic_header_crc32c) {
1154 max_rx_pdu_len += ISCSI_HEADER_DIGEST_SIZE;
1155 max_tx_pdu_len += ISCSI_HEADER_DIGEST_SIZE;
1156 }
1157 if (ic->ic_data_crc32c) {
1158 max_rx_pdu_len += ISCSI_DATA_DIGEST_SIZE;
1159 max_tx_pdu_len += ISCSI_DATA_DIGEST_SIZE;
1160 }
1161
1162 inp = sotoinpcb(so);
1163 INP_WLOCK(inp);
1164 tp = intotcpcb(inp);
1165 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1166 INP_WUNLOCK(inp);
1167 error = ENOTCONN;
1168 goto out;
1169 }
1170
1171 /*
1172 * socket could not have been "unoffloaded" if here.
1173 */
1174 MPASS(tp->t_flags & TF_TOE);
1175 MPASS(tp->tod != NULL);
1176 MPASS(tp->t_toe != NULL);
1177 toep = tp->t_toe;
1178 MPASS(toep->vi->adapter == icc->sc);
1179
1180 if (ulp_mode(toep) != ULP_MODE_NONE) {
1181 INP_WUNLOCK(inp);
1182 error = EINVAL;
1183 goto out;
1184 }
1185
1186 icc->toep = toep;
1187
1188 icc->ulp_submode = 0;
1189 if (ic->ic_header_crc32c)
1190 icc->ulp_submode |= ULP_CRC_HEADER;
1191 if (ic->ic_data_crc32c)
1192 icc->ulp_submode |= ULP_CRC_DATA;
1193
1194 if (icc->sc->tt.iso && chip_id(icc->sc) >= CHELSIO_T5 &&
1195 !is_memfree(icc->sc)) {
1196 max_iso_payload = rounddown(CXGBEI_MAX_ISO_PAYLOAD,
1197 tp->t_maxseg);
1198 max_iso_pdus = max_iso_payload / max_tx_pdu_len;
1199 ic->ic_hw_isomax = max_iso_pdus *
1200 ic->ic_max_send_data_segment_length;
1201 } else
1202 max_iso_pdus = 1;
1203
1204 toep->params.ulp_mode = ULP_MODE_ISCSI;
1205 toep->ulpcb = icc;
1206
1207 send_iscsi_flowc_wr(icc->sc, toep,
1208 roundup(max_iso_pdus * max_tx_pdu_len, tp->t_maxseg));
1209 set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode);
1210 INP_WUNLOCK(inp);
1211
1212 error = kthread_add(icl_cxgbei_tx_main, icc, NULL, &icc->tx_thread, 0,
1213 0, "%stx (cxgbei)", ic->ic_name);
1214 if (error != 0)
1215 goto out;
1216
1217 error = kthread_add(icl_cxgbei_rx_main, icc, NULL, &icc->rx_thread, 0,
1218 0, "%srx (cxgbei)", ic->ic_name);
1219 if (error != 0)
1220 goto out;
1221
1222 error = icl_cxgbei_setsockopt(ic, so, max_tx_pdu_len, max_rx_pdu_len);
1223 out:
1224 if (error != 0)
1225 icl_cxgbei_conn_close(ic);
1226 return (error);
1227 }
1228
1229 void
icl_cxgbei_conn_close(struct icl_conn * ic)1230 icl_cxgbei_conn_close(struct icl_conn *ic)
1231 {
1232 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1233 struct icl_pdu *ip;
1234 struct socket *so;
1235 struct sockbuf *sb;
1236 struct inpcb *inp;
1237 struct toepcb *toep = icc->toep;
1238
1239 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
1240 ICL_CONN_LOCK_ASSERT_NOT(ic);
1241
1242 ICL_CONN_LOCK(ic);
1243 so = ic->ic_socket;
1244 if (ic->ic_disconnecting || so == NULL) {
1245 CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p",
1246 __func__, icc, ic->ic_disconnecting, so);
1247 ICL_CONN_UNLOCK(ic);
1248 return;
1249 }
1250 ic->ic_disconnecting = true;
1251
1252 #ifdef DIAGNOSTIC
1253 KASSERT(ic->ic_outstanding_pdus == 0,
1254 ("destroying session with %d outstanding PDUs",
1255 ic->ic_outstanding_pdus));
1256 #endif
1257
1258 CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1,
1259 icc);
1260
1261 /*
1262 * Wait for the transmit thread to stop processing
1263 * this connection.
1264 */
1265 if (icc->tx_thread != NULL) {
1266 wakeup(&icc->tx_active);
1267 mtx_sleep(icc->tx_thread, ic->ic_lock, 0, "conclo", 0);
1268 }
1269
1270 /* Discard PDUs queued for TX. */
1271 while (!STAILQ_EMPTY(&icc->sent_pdus)) {
1272 ip = STAILQ_FIRST(&icc->sent_pdus);
1273 STAILQ_REMOVE_HEAD(&icc->sent_pdus, ip_next);
1274 icl_cxgbei_pdu_done(ip, ENOTCONN);
1275 }
1276 ICL_CONN_UNLOCK(ic);
1277
1278 inp = sotoinpcb(so);
1279 sb = &so->so_rcv;
1280
1281 /*
1282 * Wait for the receive thread to stop processing this
1283 * connection.
1284 */
1285 SOCKBUF_LOCK(sb);
1286 if (icc->rx_thread != NULL) {
1287 icc->rx_exiting = true;
1288 wakeup(&icc->rx_active);
1289 mtx_sleep(icc->rx_thread, SOCKBUF_MTX(sb), 0, "conclo", 0);
1290 }
1291
1292 /*
1293 * Discard received PDUs not passed to the iSCSI layer.
1294 */
1295 while (!STAILQ_EMPTY(&icc->rcvd_pdus)) {
1296 ip = STAILQ_FIRST(&icc->rcvd_pdus);
1297 STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next);
1298 icl_cxgbei_pdu_done(ip, ENOTCONN);
1299 }
1300 SOCKBUF_UNLOCK(sb);
1301
1302 INP_WLOCK(inp);
1303 if (toep != NULL) { /* NULL if connection was never offloaded. */
1304 toep->ulpcb = NULL;
1305
1306 /* Discard mbufs queued for TX. */
1307 mbufq_drain(&toep->ulp_pduq);
1308
1309 /*
1310 * Grab a reference to use when waiting for the final
1311 * CPL to be received. If toep->inp is NULL, then
1312 * final_cpl_received() has already been called (e.g.
1313 * due to the peer sending a RST).
1314 */
1315 if (toep->inp != NULL) {
1316 toep = hold_toepcb(toep);
1317 toep->flags |= TPF_WAITING_FOR_FINAL;
1318 } else
1319 toep = NULL;
1320 }
1321 INP_WUNLOCK(inp);
1322
1323 ICL_CONN_LOCK(ic);
1324 ic->ic_socket = NULL;
1325 ICL_CONN_UNLOCK(ic);
1326
1327 /*
1328 * XXXNP: we should send RST instead of FIN when PDUs held in various
1329 * queues were purged instead of delivered reliably but soabort isn't
1330 * really general purpose and wouldn't do the right thing here.
1331 */
1332 soclose(so);
1333
1334 /*
1335 * Wait for the socket to fully close. This ensures any
1336 * pending received data has been received (and in particular,
1337 * any data that would be received by DDP has been handled).
1338 * Callers assume that it is safe to free buffers for tasks
1339 * and transfers after this function returns.
1340 */
1341 if (toep != NULL) {
1342 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep);
1343
1344 mtx_lock(lock);
1345 while ((toep->flags & TPF_WAITING_FOR_FINAL) != 0)
1346 mtx_sleep(toep, lock, PSOCK, "conclo2", 0);
1347 mtx_unlock(lock);
1348 free_toepcb(toep);
1349 }
1350 }
1351
1352 static void
cxgbei_insert_cmp(struct icl_cxgbei_conn * icc,struct cxgbei_cmp * cmp,uint32_t tt)1353 cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp,
1354 uint32_t tt)
1355 {
1356 #ifdef INVARIANTS
1357 struct cxgbei_cmp *cmp2;
1358 #endif
1359
1360 cmp->tt = tt;
1361
1362 mtx_lock(&icc->cmp_lock);
1363 #ifdef INVARIANTS
1364 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) {
1365 KASSERT(cmp2->tt != tt, ("%s: duplicate cmp", __func__));
1366 }
1367 #endif
1368 LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link);
1369 mtx_unlock(&icc->cmp_lock);
1370 }
1371
1372 struct cxgbei_cmp *
cxgbei_find_cmp(struct icl_cxgbei_conn * icc,uint32_t tt)1373 cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt)
1374 {
1375 struct cxgbei_cmp *cmp;
1376
1377 mtx_lock(&icc->cmp_lock);
1378 LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) {
1379 if (cmp->tt == tt)
1380 break;
1381 }
1382 mtx_unlock(&icc->cmp_lock);
1383 return (cmp);
1384 }
1385
1386 static void
cxgbei_rm_cmp(struct icl_cxgbei_conn * icc,struct cxgbei_cmp * cmp)1387 cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp)
1388 {
1389 #ifdef INVARIANTS
1390 struct cxgbei_cmp *cmp2;
1391 #endif
1392
1393 mtx_lock(&icc->cmp_lock);
1394
1395 #ifdef INVARIANTS
1396 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) {
1397 if (cmp2 == cmp)
1398 goto found;
1399 }
1400 panic("%s: could not find cmp", __func__);
1401 found:
1402 #endif
1403 LIST_REMOVE(cmp, link);
1404 mtx_unlock(&icc->cmp_lock);
1405 }
1406
1407 int
icl_cxgbei_conn_task_setup(struct icl_conn * ic,struct icl_pdu * ip,struct ccb_scsiio * csio,uint32_t * ittp,void ** arg)1408 icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
1409 struct ccb_scsiio *csio, uint32_t *ittp, void **arg)
1410 {
1411 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1412 struct toepcb *toep = icc->toep;
1413 struct adapter *sc = icc->sc;
1414 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
1415 struct ppod_region *pr = &ci->pr;
1416 struct cxgbei_ddp_state *ddp;
1417 struct ppod_reservation *prsv;
1418 struct inpcb *inp;
1419 struct mbufq mq;
1420 uint32_t itt;
1421 int rc = 0;
1422
1423 ICL_CONN_LOCK_ASSERT(ic);
1424
1425 /* This is for the offload driver's state. Must not be set already. */
1426 MPASS(arg != NULL);
1427 MPASS(*arg == NULL);
1428
1429 if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN ||
1430 csio->dxfer_len < ci->ddp_threshold || ic->ic_disconnecting ||
1431 ic->ic_socket == NULL) {
1432 no_ddp:
1433 /*
1434 * No DDP for this I/O. Allocate an ITT (based on the one
1435 * passed in) that cannot be a valid hardware DDP tag in the
1436 * iSCSI region.
1437 */
1438 itt = *ittp & M_PPOD_TAG;
1439 itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit;
1440 *ittp = htobe32(itt);
1441 MPASS(*arg == NULL); /* State is maintained for DDP only. */
1442 if (rc != 0)
1443 counter_u64_add(
1444 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1);
1445 return (0);
1446 }
1447
1448 /*
1449 * Reserve resources for DDP, update the itt that should be used in the
1450 * PDU, and save DDP specific state for this I/O in *arg.
1451 */
1452 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1453 if (ddp == NULL) {
1454 rc = ENOMEM;
1455 goto no_ddp;
1456 }
1457 prsv = &ddp->prsv;
1458
1459 mbufq_init(&mq, INT_MAX);
1460 switch (csio->ccb_h.flags & CAM_DATA_MASK) {
1461 case CAM_DATA_BIO:
1462 rc = t4_alloc_page_pods_for_bio(pr,
1463 (struct bio *)csio->data_ptr, prsv);
1464 if (rc != 0) {
1465 free(ddp, M_CXGBEI);
1466 goto no_ddp;
1467 }
1468
1469 rc = t4_write_page_pods_for_bio(sc, toep, prsv,
1470 (struct bio *)csio->data_ptr, &mq);
1471 if (__predict_false(rc != 0)) {
1472 mbufq_drain(&mq);
1473 t4_free_page_pods(prsv);
1474 free(ddp, M_CXGBEI);
1475 goto no_ddp;
1476 }
1477 break;
1478 case CAM_DATA_VADDR:
1479 rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
1480 csio->dxfer_len, prsv);
1481 if (rc != 0) {
1482 free(ddp, M_CXGBEI);
1483 goto no_ddp;
1484 }
1485
1486 rc = t4_write_page_pods_for_buf(sc, toep, prsv,
1487 (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq);
1488 if (__predict_false(rc != 0)) {
1489 mbufq_drain(&mq);
1490 t4_free_page_pods(prsv);
1491 free(ddp, M_CXGBEI);
1492 goto no_ddp;
1493 }
1494 break;
1495 default:
1496 free(ddp, M_CXGBEI);
1497 rc = EINVAL;
1498 goto no_ddp;
1499 }
1500
1501 /*
1502 * Do not get inp from toep->inp as the toepcb might have
1503 * detached already.
1504 */
1505 inp = sotoinpcb(ic->ic_socket);
1506 INP_WLOCK(inp);
1507 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1508 INP_WUNLOCK(inp);
1509 mbufq_drain(&mq);
1510 t4_free_page_pods(prsv);
1511 free(ddp, M_CXGBEI);
1512 goto no_ddp;
1513 }
1514 mbufq_concat(&toep->ulp_pduq, &mq);
1515 INP_WUNLOCK(inp);
1516
1517 ddp->cmp.last_datasn = -1;
1518 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1519 *ittp = htobe32(prsv->prsv_tag);
1520 *arg = prsv;
1521 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1);
1522 return (0);
1523 }
1524
1525 void
icl_cxgbei_conn_task_done(struct icl_conn * ic,void * arg)1526 icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg)
1527 {
1528
1529 if (arg != NULL) {
1530 struct cxgbei_ddp_state *ddp = arg;
1531
1532 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp);
1533 t4_free_page_pods(&ddp->prsv);
1534 free(ddp, M_CXGBEI);
1535 }
1536 }
1537
1538 static inline bool
ddp_sgl_check(struct ctl_sg_entry * sg,int entries,int xferlen)1539 ddp_sgl_check(struct ctl_sg_entry *sg, int entries, int xferlen)
1540 {
1541 #ifdef INVARIANTS
1542 int total_len = 0;
1543 #endif
1544
1545 MPASS(entries > 0);
1546 if (((vm_offset_t)sg[--entries].addr & 3U) != 0)
1547 return (false);
1548
1549 #ifdef INVARIANTS
1550 total_len += sg[entries].len;
1551 #endif
1552
1553 while (--entries >= 0) {
1554 if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 ||
1555 (sg[entries].len % PAGE_SIZE) != 0)
1556 return (false);
1557 #ifdef INVARIANTS
1558 total_len += sg[entries].len;
1559 #endif
1560 }
1561
1562 MPASS(total_len == xferlen);
1563 return (true);
1564 }
1565
1566 /* XXXNP: PDU should be passed in as parameter, like on the initiator. */
1567 #define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr)
1568 #define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr)
1569
1570 int
icl_cxgbei_conn_transfer_setup(struct icl_conn * ic,union ctl_io * io,uint32_t * tttp,void ** arg)1571 icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
1572 uint32_t *tttp, void **arg)
1573 {
1574 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1575 struct toepcb *toep = icc->toep;
1576 struct ctl_scsiio *ctsio = &io->scsiio;
1577 struct adapter *sc = icc->sc;
1578 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
1579 struct ppod_region *pr = &ci->pr;
1580 struct cxgbei_ddp_state *ddp;
1581 struct ppod_reservation *prsv;
1582 struct ctl_sg_entry *sgl, sg_entry;
1583 struct inpcb *inp;
1584 struct mbufq mq;
1585 int sg_entries = ctsio->kern_sg_entries;
1586 uint32_t ttt;
1587 int xferlen, rc = 0, alias;
1588
1589 /* This is for the offload driver's state. Must not be set already. */
1590 MPASS(arg != NULL);
1591 MPASS(*arg == NULL);
1592
1593 if (ctsio->ext_data_filled == 0) {
1594 int first_burst;
1595 struct icl_pdu *ip = io_to_request_pdu(io);
1596 #ifdef INVARIANTS
1597 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
1598
1599 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
1600 MPASS(ic == ip->ip_conn);
1601 MPASS(ip->ip_bhs_mbuf != NULL);
1602 #endif
1603 first_burst = icl_pdu_data_segment_length(ip);
1604
1605 /*
1606 * Note that ICL calls conn_transfer_setup even if the first
1607 * burst had everything and there's nothing left to transfer.
1608 *
1609 * NB: The CTL frontend might have provided a buffer
1610 * whose length (kern_data_len) is smaller than the
1611 * FirstBurstLength of unsolicited data. Treat those
1612 * as an empty transfer.
1613 */
1614 xferlen = ctsio->kern_data_len;
1615 if (xferlen < first_burst ||
1616 xferlen - first_burst < ci->ddp_threshold) {
1617 no_ddp:
1618 /*
1619 * No DDP for this transfer. Allocate a TTT (based on
1620 * the one passed in) that cannot be a valid hardware
1621 * DDP tag in the iSCSI region.
1622 */
1623 ttt = *tttp & M_PPOD_TAG;
1624 ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit;
1625 *tttp = htobe32(ttt);
1626 MPASS(io_to_ddp_state(io) == NULL);
1627 if (rc != 0)
1628 counter_u64_add(
1629 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1);
1630 return (0);
1631 }
1632
1633 if (sg_entries == 0) {
1634 sgl = &sg_entry;
1635 sgl->len = xferlen;
1636 sgl->addr = (void *)ctsio->kern_data_ptr;
1637 sg_entries = 1;
1638 } else
1639 sgl = (void *)ctsio->kern_data_ptr;
1640
1641 if (!ddp_sgl_check(sgl, sg_entries, xferlen))
1642 goto no_ddp;
1643
1644 /*
1645 * Reserve resources for DDP, update the ttt that should be used
1646 * in the PDU, and save DDP specific state for this I/O.
1647 */
1648 MPASS(io_to_ddp_state(io) == NULL);
1649 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1650 if (ddp == NULL) {
1651 rc = ENOMEM;
1652 goto no_ddp;
1653 }
1654 prsv = &ddp->prsv;
1655
1656 rc = t4_alloc_page_pods_for_sgl(pr, sgl, sg_entries, prsv);
1657 if (rc != 0) {
1658 free(ddp, M_CXGBEI);
1659 goto no_ddp;
1660 }
1661
1662 mbufq_init(&mq, INT_MAX);
1663 rc = t4_write_page_pods_for_sgl(sc, toep, prsv, sgl, sg_entries,
1664 xferlen, &mq);
1665 if (__predict_false(rc != 0)) {
1666 mbufq_drain(&mq);
1667 t4_free_page_pods(prsv);
1668 free(ddp, M_CXGBEI);
1669 goto no_ddp;
1670 }
1671
1672 /*
1673 * Do not get inp from toep->inp as the toepcb might
1674 * have detached already.
1675 */
1676 ICL_CONN_LOCK(ic);
1677 if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1678 ICL_CONN_UNLOCK(ic);
1679 mbufq_drain(&mq);
1680 t4_free_page_pods(prsv);
1681 free(ddp, M_CXGBEI);
1682 return (ECONNRESET);
1683 }
1684 inp = sotoinpcb(ic->ic_socket);
1685 INP_WLOCK(inp);
1686 ICL_CONN_UNLOCK(ic);
1687 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1688 INP_WUNLOCK(inp);
1689 mbufq_drain(&mq);
1690 t4_free_page_pods(prsv);
1691 free(ddp, M_CXGBEI);
1692 return (ECONNRESET);
1693 }
1694 mbufq_concat(&toep->ulp_pduq, &mq);
1695 INP_WUNLOCK(inp);
1696
1697 ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset +
1698 first_burst;
1699 ddp->cmp.last_datasn = -1;
1700 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1701 *tttp = htobe32(prsv->prsv_tag);
1702 io_to_ddp_state(io) = ddp;
1703 *arg = ctsio;
1704 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1);
1705 return (0);
1706 }
1707
1708 /*
1709 * In the middle of an I/O. A non-NULL page pod reservation indicates
1710 * that a DDP buffer is being used for the I/O.
1711 */
1712 ddp = io_to_ddp_state(ctsio);
1713 if (ddp == NULL)
1714 goto no_ddp;
1715 prsv = &ddp->prsv;
1716
1717 alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift;
1718 alias++;
1719 prsv->prsv_tag &= ~pr->pr_alias_mask;
1720 prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask;
1721
1722 ddp->cmp.last_datasn = -1;
1723 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1724 *tttp = htobe32(prsv->prsv_tag);
1725 *arg = ctsio;
1726
1727 return (0);
1728 }
1729
1730 void
icl_cxgbei_conn_transfer_done(struct icl_conn * ic,void * arg)1731 icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg)
1732 {
1733 struct ctl_scsiio *ctsio = arg;
1734
1735 if (ctsio != NULL) {
1736 struct cxgbei_ddp_state *ddp;
1737
1738 ddp = io_to_ddp_state(ctsio);
1739 MPASS(ddp != NULL);
1740
1741 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp);
1742 if (ctsio->kern_data_len == ctsio->ext_data_filled ||
1743 ic->ic_disconnecting) {
1744 t4_free_page_pods(&ddp->prsv);
1745 free(ddp, M_CXGBEI);
1746 io_to_ddp_state(ctsio) = NULL;
1747 }
1748 }
1749 }
1750
1751 #ifdef COMPAT_FREEBSD13
1752 static void
cxgbei_limits(struct adapter * sc,void * arg)1753 cxgbei_limits(struct adapter *sc, void *arg)
1754 {
1755 struct icl_drv_limits *idl = arg;
1756 struct cxgbei_data *ci;
1757 int max_dsl;
1758
1759 if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0)
1760 return;
1761
1762 if (uld_active(sc, ULD_ISCSI)) {
1763 ci = sc->iscsi_ulp_softc;
1764 MPASS(ci != NULL);
1765
1766
1767 max_dsl = ci->max_rx_data_len;
1768 if (idl->idl_max_recv_data_segment_length > max_dsl)
1769 idl->idl_max_recv_data_segment_length = max_dsl;
1770
1771 max_dsl = ci->max_tx_data_len;
1772 if (idl->idl_max_send_data_segment_length > max_dsl)
1773 idl->idl_max_send_data_segment_length = max_dsl;
1774 }
1775
1776 end_synchronized_op(sc, LOCK_HELD);
1777 }
1778 #endif
1779
1780 static int
cxgbei_limits_fd(struct icl_drv_limits * idl,int fd)1781 cxgbei_limits_fd(struct icl_drv_limits *idl, int fd)
1782 {
1783 struct find_ofld_adapter_rr fa;
1784 struct file *fp;
1785 struct socket *so;
1786 struct adapter *sc;
1787 struct cxgbei_data *ci;
1788 cap_rights_t rights;
1789 int error;
1790
1791 error = fget(curthread, fd,
1792 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
1793 if (error != 0)
1794 return (error);
1795 if (fp->f_type != DTYPE_SOCKET) {
1796 fdrop(fp, curthread);
1797 return (EINVAL);
1798 }
1799 so = fp->f_data;
1800 if (so->so_type != SOCK_STREAM ||
1801 so->so_proto->pr_protocol != IPPROTO_TCP) {
1802 fdrop(fp, curthread);
1803 return (EINVAL);
1804 }
1805
1806 /* Find the adapter offloading this socket. */
1807 fa.sc = NULL;
1808 fa.so = so;
1809 t4_iterate(find_offload_adapter, &fa);
1810 if (fa.sc == NULL) {
1811 fdrop(fp, curthread);
1812 return (ENXIO);
1813 }
1814 fdrop(fp, curthread);
1815
1816 sc = fa.sc;
1817 error = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims");
1818 if (error != 0)
1819 return (error);
1820
1821 if (uld_active(sc, ULD_ISCSI)) {
1822 ci = sc->iscsi_ulp_softc;
1823 MPASS(ci != NULL);
1824
1825 idl->idl_max_recv_data_segment_length = ci->max_rx_data_len;
1826 idl->idl_max_send_data_segment_length = ci->max_tx_data_len;
1827 } else
1828 error = ENXIO;
1829
1830 end_synchronized_op(sc, LOCK_HELD);
1831
1832 return (error);
1833 }
1834
1835 static int
icl_cxgbei_limits(struct icl_drv_limits * idl,int socket)1836 icl_cxgbei_limits(struct icl_drv_limits *idl, int socket)
1837 {
1838
1839 /* Maximum allowed by the RFC. cxgbei_limits will clip them. */
1840 idl->idl_max_recv_data_segment_length = (1 << 24) - 1;
1841 idl->idl_max_send_data_segment_length = (1 << 24) - 1;
1842
1843 /* These are somewhat arbitrary. */
1844 idl->idl_max_burst_length = max_burst_length;
1845 idl->idl_first_burst_length = first_burst_length;
1846
1847 #ifdef COMPAT_FREEBSD13
1848 if (socket == 0) {
1849 t4_iterate(cxgbei_limits, idl);
1850 return (0);
1851 }
1852 #endif
1853
1854 return (cxgbei_limits_fd(idl, socket));
1855 }
1856
1857 int
icl_cxgbei_mod_load(void)1858 icl_cxgbei_mod_load(void)
1859 {
1860 int rc;
1861
1862 refcount_init(&icl_cxgbei_ncons, 0);
1863
1864 rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits,
1865 icl_cxgbei_new_conn);
1866
1867 return (rc);
1868 }
1869
1870 int
icl_cxgbei_mod_unload(void)1871 icl_cxgbei_mod_unload(void)
1872 {
1873
1874 if (icl_cxgbei_ncons != 0)
1875 return (EBUSY);
1876
1877 icl_unregister("cxgbei", false);
1878
1879 return (0);
1880 }
1881 #endif
1882