1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2017-2018 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: John Baldwin <jhb@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_inet.h"
31 #include "opt_kern_tls.h"
32
33 #include <sys/cdefs.h>
34 #ifdef KERN_TLS
35 #include <sys/param.h>
36 #include <sys/ktr.h>
37 #include <sys/ktls.h>
38 #include <sys/sglist.h>
39 #include <sys/socket.h>
40 #include <sys/socketvar.h>
41 #include <sys/systm.h>
42 #include <netinet/in.h>
43 #include <netinet/in_pcb.h>
44 #include <netinet/tcp_var.h>
45 #include <netinet/toecore.h>
46 #include <opencrypto/cryptodev.h>
47 #include <opencrypto/xform.h>
48
49 #ifdef TCP_OFFLOAD
50 #include "common/common.h"
51 #include "common/t4_tcb.h"
52 #include "crypto/t4_crypto.h"
53 #include "tom/t4_tom_l2t.h"
54 #include "tom/t4_tom.h"
55
56 /*
57 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while
58 * the mbuf is in the ulp_pdu_reclaimq.
59 */
60 #define tls_tcp_seq PH_loc.thirtytwo[0]
61
62 static void
t4_set_tls_tcb_field(struct toepcb * toep,uint16_t word,uint64_t mask,uint64_t val)63 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
64 uint64_t val)
65 {
66 struct adapter *sc = td_adapter(toep->td);
67
68 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0);
69 }
70
71 /* TLS and DTLS common routines */
72 bool
can_tls_offload(struct adapter * sc)73 can_tls_offload(struct adapter *sc)
74 {
75
76 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS);
77 }
78
79 int
tls_tx_key(struct toepcb * toep)80 tls_tx_key(struct toepcb *toep)
81 {
82 struct tls_ofld_info *tls_ofld = &toep->tls;
83
84 return (tls_ofld->tx_key_addr >= 0);
85 }
86
87 /* Set TLS Key-Id in TCB */
88 static void
t4_set_tls_keyid(struct toepcb * toep,unsigned int key_id)89 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id)
90 {
91
92 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG,
93 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG),
94 V_TCB_RX_TLS_KEY_TAG(key_id));
95 }
96
97 /* Clear TF_RX_QUIESCE to re-enable receive. */
98 static void
t4_clear_rx_quiesce(struct toepcb * toep)99 t4_clear_rx_quiesce(struct toepcb *toep)
100 {
101
102 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
103 }
104
105 static void
tls_clr_ofld_mode(struct toepcb * toep)106 tls_clr_ofld_mode(struct toepcb *toep)
107 {
108
109 tls_stop_handshake_timer(toep);
110
111 KASSERT(toep->tls.rx_key_addr == -1,
112 ("%s: tid %d has RX key", __func__, toep->tid));
113
114 /* Switch to plain TOE mode. */
115 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
116 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)),
117 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0)));
118 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE,
119 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE));
120 t4_clear_rx_quiesce(toep);
121
122 toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED);
123 toep->params.ulp_mode = ULP_MODE_NONE;
124 }
125
126 /* TLS/DTLS content type for CPL SFO */
127 static inline unsigned char
tls_content_type(unsigned char content_type)128 tls_content_type(unsigned char content_type)
129 {
130 switch (content_type) {
131 case CONTENT_TYPE_CCS:
132 return CPL_TX_TLS_SFO_TYPE_CCS;
133 case CONTENT_TYPE_ALERT:
134 return CPL_TX_TLS_SFO_TYPE_ALERT;
135 case CONTENT_TYPE_HANDSHAKE:
136 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
137 case CONTENT_TYPE_APP_DATA:
138 return CPL_TX_TLS_SFO_TYPE_DATA;
139 default:
140 return CPL_TX_TLS_SFO_TYPE_CUSTOM;
141 }
142 }
143
144 /* TLS Key memory management */
145 static void
clear_tls_keyid(struct toepcb * toep)146 clear_tls_keyid(struct toepcb *toep)
147 {
148 struct tls_ofld_info *tls_ofld = &toep->tls;
149 struct adapter *sc = td_adapter(toep->td);
150
151 if (tls_ofld->rx_key_addr >= 0) {
152 t4_free_tls_keyid(sc, tls_ofld->rx_key_addr);
153 tls_ofld->rx_key_addr = -1;
154 }
155 if (tls_ofld->tx_key_addr >= 0) {
156 t4_free_tls_keyid(sc, tls_ofld->tx_key_addr);
157 tls_ofld->tx_key_addr = -1;
158 }
159 }
160
161 static int
get_tp_plen_max(struct ktls_session * tls)162 get_tp_plen_max(struct ktls_session *tls)
163 {
164 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448;
165
166 return (tls->params.max_frame_len <= 8192 ? plen : FC_TP_PLEN_MAX);
167 }
168
169 /* Send request to get the key-id */
170 static int
tls_program_key_id(struct toepcb * toep,struct ktls_session * tls,int direction)171 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls,
172 int direction)
173 {
174 struct tls_ofld_info *tls_ofld = &toep->tls;
175 struct adapter *sc = td_adapter(toep->td);
176 struct ofld_tx_sdesc *txsd;
177 int keyid;
178 struct wrqe *wr;
179 struct tls_key_req *kwr;
180 struct tls_keyctx *kctx;
181
182 #ifdef INVARIANTS
183 int kwrlen, kctxlen, len;
184
185 kwrlen = sizeof(*kwr);
186 kctxlen = roundup2(sizeof(*kctx), 32);
187 len = roundup2(kwrlen + kctxlen, 16);
188 MPASS(TLS_KEY_WR_SZ == len);
189 #endif
190 if (toep->txsd_avail == 0)
191 return (EAGAIN);
192
193 if ((keyid = t4_alloc_tls_keyid(sc)) < 0) {
194 return (ENOSPC);
195 }
196
197 wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq);
198 if (wr == NULL) {
199 t4_free_tls_keyid(sc, keyid);
200 return (ENOMEM);
201 }
202 kwr = wrtod(wr);
203 memset(kwr, 0, TLS_KEY_WR_SZ);
204
205 t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid,
206 kwr);
207 kctx = (struct tls_keyctx *)(kwr + 1);
208 if (direction == KTLS_TX)
209 tls_ofld->tx_key_addr = keyid;
210 else
211 tls_ofld->rx_key_addr = keyid;
212 t4_tls_key_ctx(tls, direction, kctx);
213
214 txsd = &toep->txsd[toep->txsd_pidx];
215 txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16);
216 txsd->plen = 0;
217 toep->tx_credits -= txsd->tx_credits;
218 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
219 toep->txsd_pidx = 0;
220 toep->txsd_avail--;
221
222 t4_wrq_tx(sc, wr);
223
224 return (0);
225 }
226
227 /*
228 * In some cases a client connection can hang without sending the
229 * ServerHelloDone message from the NIC to the host. Send a dummy
230 * RX_DATA_ACK with RX_MODULATE to unstick the connection.
231 */
232 static void
tls_send_handshake_ack(void * arg)233 tls_send_handshake_ack(void *arg)
234 {
235 struct toepcb *toep = arg;
236 struct tls_ofld_info *tls_ofld = &toep->tls;
237 struct adapter *sc = td_adapter(toep->td);
238
239 /* Bail without rescheduling if the connection has closed. */
240 if ((toep->flags & (TPF_FIN_SENT | TPF_ABORT_SHUTDOWN)) != 0)
241 return;
242
243 /*
244 * If this connection has timed out without receiving more
245 * data, downgrade to plain TOE mode and don't re-arm the
246 * timer.
247 */
248 if (sc->tt.tls_rx_timeout != 0) {
249 struct inpcb *inp;
250 struct tcpcb *tp;
251
252 inp = toep->inp;
253 tp = intotcpcb(inp);
254 if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) {
255 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
256 toep->tid);
257 tls_clr_ofld_mode(toep);
258 return;
259 }
260 }
261
262 /*
263 * XXX: Does not have the t4_get_tcb() checks to refine the
264 * workaround.
265 */
266 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);
267
268 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
269 send_rx_modulate(sc, toep);
270 }
271
272 static void
tls_start_handshake_timer(struct toepcb * toep)273 tls_start_handshake_timer(struct toepcb *toep)
274 {
275 struct tls_ofld_info *tls_ofld = &toep->tls;
276
277 INP_WLOCK_ASSERT(toep->inp);
278 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
279 tls_send_handshake_ack, toep);
280 }
281
282 void
tls_stop_handshake_timer(struct toepcb * toep)283 tls_stop_handshake_timer(struct toepcb *toep)
284 {
285 struct tls_ofld_info *tls_ofld = &toep->tls;
286
287 INP_WLOCK_ASSERT(toep->inp);
288 callout_stop(&tls_ofld->handshake_timer);
289 }
290
291 int
tls_alloc_ktls(struct toepcb * toep,struct ktls_session * tls,int direction)292 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
293 {
294 struct adapter *sc = td_adapter(toep->td);
295 int error, explicit_iv_size, key_offset, mac_first;
296
297 if (!can_tls_offload(td_adapter(toep->td)))
298 return (EINVAL);
299 switch (ulp_mode(toep)) {
300 case ULP_MODE_TLS:
301 break;
302 case ULP_MODE_NONE:
303 case ULP_MODE_TCPDDP:
304 if (direction != KTLS_TX)
305 return (EINVAL);
306 break;
307 default:
308 return (EINVAL);
309 }
310
311 switch (tls->params.cipher_algorithm) {
312 case CRYPTO_AES_CBC:
313 /* XXX: Explicitly ignore any provided IV. */
314 switch (tls->params.cipher_key_len) {
315 case 128 / 8:
316 case 192 / 8:
317 case 256 / 8:
318 break;
319 default:
320 error = EINVAL;
321 goto clr_ofld;
322 }
323 switch (tls->params.auth_algorithm) {
324 case CRYPTO_SHA1_HMAC:
325 case CRYPTO_SHA2_256_HMAC:
326 case CRYPTO_SHA2_384_HMAC:
327 break;
328 default:
329 error = EPROTONOSUPPORT;
330 goto clr_ofld;
331 }
332 explicit_iv_size = AES_BLOCK_LEN;
333 mac_first = 1;
334 break;
335 case CRYPTO_AES_NIST_GCM_16:
336 if (tls->params.iv_len != SALT_SIZE) {
337 error = EINVAL;
338 goto clr_ofld;
339 }
340 switch (tls->params.cipher_key_len) {
341 case 128 / 8:
342 case 192 / 8:
343 case 256 / 8:
344 break;
345 default:
346 error = EINVAL;
347 goto clr_ofld;
348 }
349 explicit_iv_size = 8;
350 mac_first = 0;
351 break;
352 default:
353 error = EPROTONOSUPPORT;
354 goto clr_ofld;
355 }
356
357 /* Only TLS 1.1 and TLS 1.2 are currently supported. */
358 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
359 tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
360 tls->params.tls_vminor > TLS_MINOR_VER_TWO) {
361 error = EPROTONOSUPPORT;
362 goto clr_ofld;
363 }
364
365 /* Bail if we already have a key. */
366 if (direction == KTLS_TX) {
367 if (toep->tls.tx_key_addr != -1)
368 return (EOPNOTSUPP);
369 } else {
370 if (toep->tls.rx_key_addr != -1)
371 return (EOPNOTSUPP);
372 }
373
374 error = tls_program_key_id(toep, tls, direction);
375 if (error) {
376 if (direction == KTLS_RX)
377 goto clr_ofld;
378 return (error);
379 }
380
381 if (direction == KTLS_TX) {
382 toep->tls.scmd0.seqno_numivs =
383 (V_SCMD_SEQ_NO_CTRL(3) |
384 V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) |
385 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
386 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
387 V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) |
388 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) |
389 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) |
390 V_SCMD_IV_SIZE(explicit_iv_size / 2));
391
392 toep->tls.scmd0.ivgen_hdrlen =
393 (V_SCMD_IV_GEN_CTRL(1) |
394 V_SCMD_KEY_CTX_INLINE(0) |
395 V_SCMD_TLS_FRAG_ENABLE(1));
396
397 toep->tls.iv_len = explicit_iv_size;
398 toep->tls.frag_size = tls->params.max_frame_len;
399 toep->tls.fcplenmax = get_tp_plen_max(tls);
400 toep->tls.expn_per_ulp = tls->params.tls_hlen +
401 tls->params.tls_tlen;
402 toep->tls.pdus_per_ulp = 1;
403 toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
404 tls->params.max_frame_len;
405 toep->tls.tx_key_info_size = t4_tls_key_info_size(tls);
406 } else {
407 /* Stop timer on handshake completion */
408 tls_stop_handshake_timer(toep);
409
410 toep->flags &= ~TPF_FORCE_CREDITS;
411 toep->flags |= TPF_TLS_RECEIVE;
412 toep->tls.rx_version = tls->params.tls_vmajor << 8 |
413 tls->params.tls_vminor;
414
415 /*
416 * RX key tags are an index into the key portion of MA
417 * memory stored as an offset from the base address in
418 * units of 64 bytes.
419 */
420 key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
421 t4_set_tls_keyid(toep, key_offset / 64);
422 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
423 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
424 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
425 V_TF_TLS_CONTROL(1) |
426 V_TF_TLS_ACTIVE(1) |
427 V_TF_TLS_ENABLE(1))));
428 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
429 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
430 V_TCB_TLS_SEQ(0));
431 t4_clear_rx_quiesce(toep);
432 }
433
434 return (0);
435
436 clr_ofld:
437 if (ulp_mode(toep) == ULP_MODE_TLS) {
438 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
439 toep->tid);
440 tls_clr_ofld_mode(toep);
441 }
442 return (error);
443 }
444
445 void
tls_init_toep(struct toepcb * toep)446 tls_init_toep(struct toepcb *toep)
447 {
448 struct tls_ofld_info *tls_ofld = &toep->tls;
449
450 tls_ofld->rx_key_addr = -1;
451 tls_ofld->tx_key_addr = -1;
452 }
453
454 void
tls_establish(struct toepcb * toep)455 tls_establish(struct toepcb *toep)
456 {
457
458 /*
459 * Enable PDU extraction.
460 *
461 * XXX: Supposedly this should be done by the firmware when
462 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
463 * in practice this seems to be required.
464 */
465 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
466 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
467 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
468
469 toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED;
470
471 callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0);
472 tls_start_handshake_timer(toep);
473 }
474
475 void
tls_detach(struct toepcb * toep)476 tls_detach(struct toepcb *toep)
477 {
478
479 if (toep->flags & TPF_TLS_ESTABLISHED) {
480 tls_stop_handshake_timer(toep);
481 toep->flags &= ~TPF_TLS_ESTABLISHED;
482 }
483 }
484
485 void
tls_uninit_toep(struct toepcb * toep)486 tls_uninit_toep(struct toepcb *toep)
487 {
488
489 MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0);
490 clear_tls_keyid(toep);
491 }
492
493 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
494 #define MIN_OFLD_TLSTX_CREDITS(toep) \
495 (howmany(sizeof(struct fw_tlstx_data_wr) + \
496 sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \
497 sizeof(struct ulptx_sc_memrd) + \
498 AES_BLOCK_LEN + 1, 16))
499
500 static void
write_tlstx_wr(struct fw_tlstx_data_wr * txwr,struct toepcb * toep,unsigned int plen,unsigned int expn,uint8_t credits,int shove)501 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
502 unsigned int plen, unsigned int expn, uint8_t credits, int shove)
503 {
504 struct tls_ofld_info *tls_ofld = &toep->tls;
505 unsigned int len = plen + expn;
506
507 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) |
508 V_FW_TLSTX_DATA_WR_COMPL(1) |
509 V_FW_TLSTX_DATA_WR_IMMDLEN(0));
510 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
511 V_FW_TLSTX_DATA_WR_LEN16(credits));
512 txwr->plen = htobe32(len);
513 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) |
514 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove));
515 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(1) |
516 V_FW_TLSTX_DATA_WR_EXP(expn) |
517 V_FW_TLSTX_DATA_WR_CTXLOC(TLS_SFO_WR_CONTEXTLOC_DDR) |
518 V_FW_TLSTX_DATA_WR_IVDSGL(0) |
519 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->tx_key_info_size >> 4));
520 txwr->mfs = htobe16(tls_ofld->frag_size);
521 txwr->adjustedplen_pkd = htobe16(
522 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen));
523 txwr->expinplenmax_pkd = htobe16(
524 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp));
525 txwr->pdusinplenmax_pkd =
526 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp);
527 }
528
529 static void
write_tlstx_cpl(struct cpl_tx_tls_sfo * cpl,struct toepcb * toep,struct tls_hdr * tls_hdr,unsigned int plen,uint64_t seqno)530 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
531 struct tls_hdr *tls_hdr, unsigned int plen, uint64_t seqno)
532 {
533 struct tls_ofld_info *tls_ofld = &toep->tls;
534 int data_type, seglen;
535
536 seglen = plen;
537 data_type = tls_content_type(tls_hdr->type);
538 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) |
539 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) |
540 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen));
541 cpl->pld_len = htobe32(plen);
542 if (data_type == CPL_TX_TLS_SFO_TYPE_CUSTOM)
543 cpl->type_protover = htobe32(
544 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type));
545 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs |
546 V_SCMD_NUM_IVS(1));
547 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen);
548 cpl->scmd1 = htobe64(seqno);
549 }
550
551 static int
count_ext_pgs_segs(struct mbuf * m)552 count_ext_pgs_segs(struct mbuf *m)
553 {
554 vm_paddr_t nextpa;
555 u_int i, nsegs;
556
557 MPASS(m->m_epg_npgs > 0);
558 nsegs = 1;
559 nextpa = m->m_epg_pa[0] + PAGE_SIZE;
560 for (i = 1; i < m->m_epg_npgs; i++) {
561 if (nextpa != m->m_epg_pa[i])
562 nsegs++;
563 nextpa = m->m_epg_pa[i] + PAGE_SIZE;
564 }
565 return (nsegs);
566 }
567
568 static void
write_ktlstx_sgl(void * dst,struct mbuf * m,int nsegs)569 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
570 {
571 struct ulptx_sgl *usgl = dst;
572 vm_paddr_t pa;
573 uint32_t len;
574 int i, j;
575
576 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
577
578 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
579 V_ULPTX_NSGE(nsegs));
580
581 /* Figure out the first S/G length. */
582 pa = m->m_epg_pa[0] + m->m_epg_1st_off;
583 usgl->addr0 = htobe64(pa);
584 len = m_epg_pagelen(m, 0, m->m_epg_1st_off);
585 pa += len;
586 for (i = 1; i < m->m_epg_npgs; i++) {
587 if (m->m_epg_pa[i] != pa)
588 break;
589 len += m_epg_pagelen(m, i, 0);
590 pa += m_epg_pagelen(m, i, 0);
591 }
592 usgl->len0 = htobe32(len);
593 #ifdef INVARIANTS
594 nsegs--;
595 #endif
596
597 j = -1;
598 for (; i < m->m_epg_npgs; i++) {
599 if (j == -1 || m->m_epg_pa[i] != pa) {
600 if (j >= 0)
601 usgl->sge[j / 2].len[j & 1] = htobe32(len);
602 j++;
603 #ifdef INVARIANTS
604 nsegs--;
605 #endif
606 pa = m->m_epg_pa[i];
607 usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
608 len = m_epg_pagelen(m, i, 0);
609 pa += len;
610 } else {
611 len += m_epg_pagelen(m, i, 0);
612 pa += m_epg_pagelen(m, i, 0);
613 }
614 }
615 if (j >= 0) {
616 usgl->sge[j / 2].len[j & 1] = htobe32(len);
617
618 if ((j & 1) == 0)
619 usgl->sge[j / 2].len[1] = htobe32(0);
620 }
621 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
622 }
623
624 /*
625 * Similar to t4_push_frames() but handles sockets that contain TLS
626 * record mbufs.
627 */
628 void
t4_push_ktls(struct adapter * sc,struct toepcb * toep,int drop)629 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
630 {
631 struct tls_hdr *thdr;
632 struct fw_tlstx_data_wr *txwr;
633 struct cpl_tx_tls_sfo *cpl;
634 struct ulptx_idata *idata;
635 struct ulptx_sc_memrd *memrd;
636 struct wrqe *wr;
637 struct mbuf *m;
638 u_int nsegs, credits, wr_len;
639 u_int expn_size;
640 struct inpcb *inp = toep->inp;
641 struct tcpcb *tp = intotcpcb(inp);
642 struct socket *so = inp->inp_socket;
643 struct sockbuf *sb = &so->so_snd;
644 int tls_size, tx_credits, shove, sowwakeup;
645 struct ofld_tx_sdesc *txsd;
646 char *buf;
647
648 INP_WLOCK_ASSERT(inp);
649 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
650 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
651
652 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
653 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
654 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
655 KASSERT(tls_tx_key(toep),
656 ("%s: TX key not set for toep %p", __func__, toep));
657
658 #ifdef VERBOSE_TRACES
659 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
660 __func__, toep->tid, toep->flags, tp->t_flags);
661 #endif
662 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
663 return;
664
665 #ifdef RATELIMIT
666 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
667 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
668 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
669 }
670 #endif
671
672 /*
673 * This function doesn't resume by itself. Someone else must clear the
674 * flag and call this function.
675 */
676 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
677 KASSERT(drop == 0,
678 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
679 return;
680 }
681
682 txsd = &toep->txsd[toep->txsd_pidx];
683 for (;;) {
684 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
685
686 SOCKBUF_LOCK(sb);
687 sowwakeup = drop;
688 if (drop) {
689 sbdrop_locked(sb, drop);
690 drop = 0;
691 }
692
693 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb;
694
695 /*
696 * Send a FIN if requested, but only if there's no
697 * more data to send.
698 */
699 if (m == NULL && toep->flags & TPF_SEND_FIN) {
700 if (sowwakeup)
701 sowwakeup_locked(so);
702 else
703 SOCKBUF_UNLOCK(sb);
704 SOCKBUF_UNLOCK_ASSERT(sb);
705 t4_close_conn(sc, toep);
706 return;
707 }
708
709 /*
710 * If there is no ready data to send, wait until more
711 * data arrives.
712 */
713 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) {
714 if (sowwakeup)
715 sowwakeup_locked(so);
716 else
717 SOCKBUF_UNLOCK(sb);
718 SOCKBUF_UNLOCK_ASSERT(sb);
719 #ifdef VERBOSE_TRACES
720 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send",
721 __func__, toep->tid);
722 #endif
723 return;
724 }
725
726 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP",
727 __func__, m));
728 KASSERT(m->m_epg_tls != NULL,
729 ("%s: mbuf %p doesn't have TLS session", __func__, m));
730
731 /* Calculate WR length. */
732 wr_len = sizeof(struct fw_tlstx_data_wr) +
733 sizeof(struct cpl_tx_tls_sfo) +
734 sizeof(struct ulptx_idata) + sizeof(struct ulptx_sc_memrd);
735
736 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */
737 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN);
738 wr_len += AES_BLOCK_LEN;
739
740 /* Account for SGL in work request length. */
741 nsegs = count_ext_pgs_segs(m);
742 wr_len += sizeof(struct ulptx_sgl) +
743 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
744
745 /* Not enough credits for this work request. */
746 if (howmany(wr_len, 16) > tx_credits) {
747 if (sowwakeup)
748 sowwakeup_locked(so);
749 else
750 SOCKBUF_UNLOCK(sb);
751 SOCKBUF_UNLOCK_ASSERT(sb);
752 #ifdef VERBOSE_TRACES
753 CTR5(KTR_CXGBE,
754 "%s: tid %d mbuf %p requires %d credits, but only %d available",
755 __func__, toep->tid, m, howmany(wr_len, 16),
756 tx_credits);
757 #endif
758 toep->flags |= TPF_TX_SUSPENDED;
759 return;
760 }
761
762 /* Shove if there is no additional data pending. */
763 shove = ((m->m_next == NULL ||
764 (m->m_next->m_flags & M_NOTAVAIL) != 0)) &&
765 (tp->t_flags & TF_MORETOCOME) == 0;
766
767 if (sb->sb_flags & SB_AUTOSIZE &&
768 V_tcp_do_autosndbuf &&
769 sb->sb_hiwat < V_tcp_autosndbuf_max &&
770 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
771 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
772 V_tcp_autosndbuf_max);
773
774 if (!sbreserve_locked(sb, newsize, so, NULL))
775 sb->sb_flags &= ~SB_AUTOSIZE;
776 else
777 sowwakeup = 1; /* room available */
778 }
779 if (sowwakeup)
780 sowwakeup_locked(so);
781 else
782 SOCKBUF_UNLOCK(sb);
783 SOCKBUF_UNLOCK_ASSERT(sb);
784
785 if (__predict_false(toep->flags & TPF_FIN_SENT))
786 panic("%s: excess tx.", __func__);
787
788 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
789 if (wr == NULL) {
790 /* XXX: how will we recover from this? */
791 toep->flags |= TPF_TX_SUSPENDED;
792 return;
793 }
794
795 thdr = (struct tls_hdr *)&m->m_epg_hdr;
796 #ifdef VERBOSE_TRACES
797 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x",
798 __func__, toep->tid, m->m_epg_seqno, thdr->type,
799 m->m_len);
800 #endif
801 txwr = wrtod(wr);
802 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
803 memset(txwr, 0, roundup2(wr_len, 16));
804 credits = howmany(wr_len, 16);
805 expn_size = m->m_epg_hdrlen +
806 m->m_epg_trllen;
807 tls_size = m->m_len - expn_size;
808 write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove);
809 write_tlstx_cpl(cpl, toep, thdr, tls_size, m->m_epg_seqno);
810
811 idata = (struct ulptx_idata *)(cpl + 1);
812 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
813 idata->len = htobe32(0);
814 memrd = (struct ulptx_sc_memrd *)(idata + 1);
815 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
816 V_ULP_TX_SC_MORE(1) |
817 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4));
818 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5);
819
820 /* Copy IV. */
821 buf = (char *)(memrd + 1);
822 memcpy(buf, thdr + 1, toep->tls.iv_len);
823 buf += AES_BLOCK_LEN;
824
825 write_ktlstx_sgl(buf, m, nsegs);
826
827 KASSERT(toep->tx_credits >= credits,
828 ("%s: not enough credits", __func__));
829
830 toep->tx_credits -= credits;
831
832 tp->snd_nxt += m->m_len;
833 tp->snd_max += m->m_len;
834
835 SOCKBUF_LOCK(sb);
836 sb->sb_sndptr = m;
837 SOCKBUF_UNLOCK(sb);
838
839 toep->flags |= TPF_TX_DATA_SENT;
840 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
841 toep->flags |= TPF_TX_SUSPENDED;
842
843 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
844 txsd->plen = m->m_len;
845 txsd->tx_credits = credits;
846 txsd++;
847 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
848 toep->txsd_pidx = 0;
849 txsd = &toep->txsd[0];
850 }
851 toep->txsd_avail--;
852
853 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
854 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len);
855
856 t4_l2t_send(sc, wr, toep->l2te);
857 }
858 }
859
860 /*
861 * For TLS data we place received mbufs received via CPL_TLS_DATA into
862 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is
863 * received, the completed PDUs are placed into the socket receive
864 * buffer.
865 *
866 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs.
867 */
868 static int
do_tls_data(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)869 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
870 {
871 struct adapter *sc = iq->adapter;
872 const struct cpl_tls_data *cpl = mtod(m, const void *);
873 unsigned int tid = GET_TID(cpl);
874 struct toepcb *toep = lookup_tid(sc, tid);
875 struct inpcb *inp = toep->inp;
876 struct tcpcb *tp;
877 int len;
878
879 /* XXX: Should this match do_rx_data instead? */
880 KASSERT(!(toep->flags & TPF_SYNQE),
881 ("%s: toep %p claims to be a synq entry", __func__, toep));
882
883 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
884
885 /* strip off CPL header */
886 m_adj(m, sizeof(*cpl));
887 len = m->m_pkthdr.len;
888
889 toep->ofld_rxq->rx_toe_tls_octets += len;
890
891 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
892 ("%s: payload length mismatch", __func__));
893
894 INP_WLOCK(inp);
895 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
896 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
897 __func__, tid, len, inp->inp_flags);
898 INP_WUNLOCK(inp);
899 m_freem(m);
900 return (0);
901 }
902
903 /* Save TCP sequence number. */
904 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq);
905
906 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
907 #ifdef INVARIANTS
908 panic("Failed to queue TLS data packet");
909 #else
910 printf("%s: Failed to queue TLS data packet\n", __func__);
911 INP_WUNLOCK(inp);
912 m_freem(m);
913 return (0);
914 #endif
915 }
916
917 tp = intotcpcb(inp);
918 tp->t_rcvtime = ticks;
919
920 #ifdef VERBOSE_TRACES
921 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
922 be32toh(cpl->seq));
923 #endif
924
925 INP_WUNLOCK(inp);
926 return (0);
927 }
928
929 static int
do_rx_tls_cmp(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)930 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
931 {
932 struct adapter *sc = iq->adapter;
933 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *);
934 struct tlsrx_hdr_pkt *tls_hdr_pkt;
935 unsigned int tid = GET_TID(cpl);
936 struct toepcb *toep = lookup_tid(sc, tid);
937 struct inpcb *inp = toep->inp;
938 struct tcpcb *tp;
939 struct socket *so;
940 struct sockbuf *sb;
941 struct mbuf *tls_data;
942 struct tls_get_record *tgr;
943 struct mbuf *control;
944 int pdu_length;
945 #if defined(KTR) || defined(INVARIANTS)
946 int len;
947 #endif
948
949 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
950 KASSERT(!(toep->flags & TPF_SYNQE),
951 ("%s: toep %p claims to be a synq entry", __func__, toep));
952
953 /* strip off CPL header */
954 m_adj(m, sizeof(*cpl));
955 #if defined(KTR) || defined(INVARIANTS)
956 len = m->m_pkthdr.len;
957 #endif
958
959 toep->ofld_rxq->rx_toe_tls_records++;
960
961 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
962 ("%s: payload length mismatch", __func__));
963
964 INP_WLOCK(inp);
965 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
966 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
967 __func__, tid, len, inp->inp_flags);
968 INP_WUNLOCK(inp);
969 m_freem(m);
970 return (0);
971 }
972
973 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
974
975 so = inp_inpcbtosocket(inp);
976 tp = intotcpcb(inp);
977
978 #ifdef VERBOSE_TRACES
979 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
980 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt);
981 #endif
982
983 tp->rcv_nxt += pdu_length;
984 KASSERT(tp->rcv_wnd >= pdu_length,
985 ("%s: negative window size", __func__));
986 tp->rcv_wnd -= pdu_length;
987
988 /* XXX: Not sure what to do about urgent data. */
989
990 /*
991 * The payload of this CPL is the TLS header followed by
992 * additional fields.
993 */
994 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt),
995 ("%s: payload too small", __func__));
996 tls_hdr_pkt = mtod(m, void *);
997
998 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
999 if (tls_data != NULL) {
1000 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
1001 ("%s: sequence mismatch", __func__));
1002 }
1003
1004 /* Report decryption errors as EBADMSG. */
1005 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) {
1006 m_freem(m);
1007 m_freem(tls_data);
1008
1009 CURVNET_SET(toep->vnet);
1010 so->so_error = EBADMSG;
1011 sorwakeup(so);
1012
1013 INP_WUNLOCK(inp);
1014 CURVNET_RESTORE();
1015
1016 return (0);
1017 }
1018
1019 /* Allocate the control message mbuf. */
1020 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD,
1021 IPPROTO_TCP);
1022 if (control == NULL) {
1023 m_freem(m);
1024 m_freem(tls_data);
1025
1026 CURVNET_SET(toep->vnet);
1027 so->so_error = ENOBUFS;
1028 sorwakeup(so);
1029
1030 INP_WUNLOCK(inp);
1031 CURVNET_RESTORE();
1032
1033 return (0);
1034 }
1035
1036 tgr = (struct tls_get_record *)
1037 CMSG_DATA(mtod(control, struct cmsghdr *));
1038 memset(tgr, 0, sizeof(*tgr));
1039 tgr->tls_type = tls_hdr_pkt->type;
1040 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8;
1041 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff;
1042
1043 m_freem(m);
1044
1045 if (tls_data != NULL) {
1046 m_last(tls_data)->m_flags |= M_EOR;
1047 tgr->tls_length = htobe16(tls_data->m_pkthdr.len);
1048 } else
1049 tgr->tls_length = 0;
1050 m = tls_data;
1051
1052 sb = &so->so_rcv;
1053 SOCKBUF_LOCK(sb);
1054
1055 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1056 struct epoch_tracker et;
1057
1058 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1059 __func__, tid, pdu_length);
1060 m_freem(m);
1061 m_freem(control);
1062 SOCKBUF_UNLOCK(sb);
1063 INP_WUNLOCK(inp);
1064
1065 CURVNET_SET(toep->vnet);
1066 NET_EPOCH_ENTER(et);
1067 INP_WLOCK(inp);
1068 tp = tcp_drop(tp, ECONNRESET);
1069 if (tp)
1070 INP_WUNLOCK(inp);
1071 NET_EPOCH_EXIT(et);
1072 CURVNET_RESTORE();
1073
1074 return (0);
1075 }
1076
1077 /*
1078 * Not all of the bytes on the wire are included in the socket buffer
1079 * (e.g. the MAC of the TLS record). However, those bytes are included
1080 * in the TCP sequence space.
1081 */
1082
1083 /* receive buffer autosize */
1084 MPASS(toep->vnet == so->so_vnet);
1085 CURVNET_SET(toep->vnet);
1086 if (sb->sb_flags & SB_AUTOSIZE &&
1087 V_tcp_do_autorcvbuf &&
1088 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1089 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) {
1090 unsigned int hiwat = sb->sb_hiwat;
1091 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
1092 V_tcp_autorcvbuf_max);
1093
1094 if (!sbreserve_locked(sb, newsize, so, NULL))
1095 sb->sb_flags &= ~SB_AUTOSIZE;
1096 }
1097
1098 sbappendcontrol_locked(sb, m, control, 0);
1099 t4_rcvd_locked(&toep->td->tod, tp);
1100
1101 sorwakeup_locked(so);
1102 SOCKBUF_UNLOCK_ASSERT(sb);
1103
1104 INP_WUNLOCK(inp);
1105 CURVNET_RESTORE();
1106 return (0);
1107 }
1108
1109 void
do_rx_data_tls(const struct cpl_rx_data * cpl,struct toepcb * toep,struct mbuf * m)1110 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep,
1111 struct mbuf *m)
1112 {
1113 struct inpcb *inp = toep->inp;
1114 struct tls_ofld_info *tls_ofld = &toep->tls;
1115 struct tls_hdr *hdr;
1116 struct tcpcb *tp;
1117 struct socket *so;
1118 struct sockbuf *sb;
1119 int len;
1120
1121 len = m->m_pkthdr.len;
1122
1123 INP_WLOCK_ASSERT(inp);
1124
1125 so = inp_inpcbtosocket(inp);
1126 tp = intotcpcb(inp);
1127 sb = &so->so_rcv;
1128 SOCKBUF_LOCK(sb);
1129 CURVNET_SET(toep->vnet);
1130
1131 tp->rcv_nxt += len;
1132 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
1133 tp->rcv_wnd -= len;
1134
1135 /* Do we have a full TLS header? */
1136 if (len < sizeof(*hdr)) {
1137 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header",
1138 __func__, toep->tid, len);
1139 so->so_error = EMSGSIZE;
1140 goto out;
1141 }
1142 hdr = mtod(m, struct tls_hdr *);
1143
1144 /* Is the header valid? */
1145 if (be16toh(hdr->version) != tls_ofld->rx_version) {
1146 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x",
1147 __func__, toep->tid, be16toh(hdr->version));
1148 so->so_error = EINVAL;
1149 goto out;
1150 }
1151 if (be16toh(hdr->length) < sizeof(*hdr)) {
1152 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u",
1153 __func__, toep->tid, be16toh(hdr->length));
1154 so->so_error = EBADMSG;
1155 goto out;
1156 }
1157
1158 /* Did we get a truncated record? */
1159 if (len < be16toh(hdr->length)) {
1160 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)",
1161 __func__, toep->tid, len, be16toh(hdr->length));
1162
1163 so->so_error = EMSGSIZE;
1164 goto out;
1165 }
1166
1167 /* Is the header type unknown? */
1168 switch (hdr->type) {
1169 case CONTENT_TYPE_CCS:
1170 case CONTENT_TYPE_ALERT:
1171 case CONTENT_TYPE_APP_DATA:
1172 case CONTENT_TYPE_HANDSHAKE:
1173 break;
1174 default:
1175 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u",
1176 __func__, toep->tid, hdr->type);
1177 so->so_error = EBADMSG;
1178 goto out;
1179 }
1180
1181 /*
1182 * Just punt. Although this could fall back to software
1183 * decryption, this case should never really happen.
1184 */
1185 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u",
1186 __func__, toep->tid, hdr->type, be16toh(hdr->length));
1187 so->so_error = EBADMSG;
1188
1189 out:
1190 sorwakeup_locked(so);
1191 SOCKBUF_UNLOCK_ASSERT(sb);
1192
1193 INP_WUNLOCK(inp);
1194 CURVNET_RESTORE();
1195
1196 m_freem(m);
1197 }
1198
1199 void
t4_tls_mod_load(void)1200 t4_tls_mod_load(void)
1201 {
1202
1203 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
1204 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
1205 }
1206
1207 void
t4_tls_mod_unload(void)1208 t4_tls_mod_unload(void)
1209 {
1210
1211 t4_register_cpl_handler(CPL_TLS_DATA, NULL);
1212 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
1213 }
1214 #endif /* TCP_OFFLOAD */
1215 #endif /* KERN_TLS */
1216