1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
5 * The Regents of the University of California. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * @(#)tcp_subr.c 8.2 (Berkeley) 5/24/95
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37 #include "opt_ipsec.h"
38 #include "opt_kern_tls.h"
39
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/arb.h>
43 #include <sys/callout.h>
44 #include <sys/eventhandler.h>
45 #ifdef TCP_HHOOK
46 #include <sys/hhook.h>
47 #endif
48 #include <sys/kernel.h>
49 #ifdef TCP_HHOOK
50 #include <sys/khelp.h>
51 #endif
52 #ifdef KERN_TLS
53 #include <sys/ktls.h>
54 #endif
55 #include <sys/qmath.h>
56 #include <sys/stats.h>
57 #include <sys/sysctl.h>
58 #include <sys/jail.h>
59 #include <sys/malloc.h>
60 #include <sys/refcount.h>
61 #include <sys/mbuf.h>
62 #include <sys/priv.h>
63 #include <sys/proc.h>
64 #include <sys/sdt.h>
65 #include <sys/socket.h>
66 #include <sys/socketvar.h>
67 #include <sys/protosw.h>
68 #include <sys/random.h>
69
70 #include <vm/uma.h>
71
72 #include <net/route.h>
73 #include <net/route/nhop.h>
74 #include <net/if.h>
75 #include <net/if_var.h>
76 #include <net/if_private.h>
77 #include <net/vnet.h>
78
79 #include <netinet/in.h>
80 #include <netinet/in_fib.h>
81 #include <netinet/in_kdtrace.h>
82 #include <netinet/in_pcb.h>
83 #include <netinet/in_systm.h>
84 #include <netinet/in_var.h>
85 #include <netinet/ip.h>
86 #include <netinet/ip_icmp.h>
87 #include <netinet/ip_var.h>
88 #ifdef INET6
89 #include <netinet/icmp6.h>
90 #include <netinet/ip6.h>
91 #include <netinet6/in6_fib.h>
92 #include <netinet6/in6_pcb.h>
93 #include <netinet6/ip6_var.h>
94 #include <netinet6/scope6_var.h>
95 #include <netinet6/nd6.h>
96 #endif
97
98 #include <netinet/tcp.h>
99 #ifdef INVARIANTS
100 #define TCPSTATES
101 #endif
102 #include <netinet/tcp_fsm.h>
103 #include <netinet/tcp_seq.h>
104 #include <netinet/tcp_timer.h>
105 #include <netinet/tcp_var.h>
106 #include <netinet/tcp_ecn.h>
107 #include <netinet/tcp_log_buf.h>
108 #include <netinet/tcp_syncache.h>
109 #include <netinet/tcp_hpts.h>
110 #include <netinet/tcp_lro.h>
111 #include <netinet/cc/cc.h>
112 #include <netinet/tcpip.h>
113 #include <netinet/tcp_fastopen.h>
114 #include <netinet/tcp_accounting.h>
115 #ifdef TCPPCAP
116 #include <netinet/tcp_pcap.h>
117 #endif
118 #ifdef TCP_OFFLOAD
119 #include <netinet/tcp_offload.h>
120 #endif
121 #include <netinet/udp.h>
122 #include <netinet/udp_var.h>
123 #ifdef INET6
124 #include <netinet6/tcp6_var.h>
125 #endif
126
127 #include <netipsec/ipsec_support.h>
128
129 #include <machine/in_cksum.h>
130 #include <crypto/siphash/siphash.h>
131
132 #include <security/mac/mac_framework.h>
133
134 #ifdef INET6
135 static ip6proto_ctlinput_t tcp6_ctlinput;
136 static udp_tun_icmp_t tcp6_ctlinput_viaudp;
137 #endif
138
139 VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
140 #ifdef INET6
141 VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
142 #endif
143
144 #ifdef TCP_SAD_DETECTION
145 /* Sack attack detection thresholds and such */
146 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack_attack,
147 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
148 "Sack Attack detection thresholds");
149 int32_t tcp_force_detection = 0;
150 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, force_detection,
151 CTLFLAG_RW,
152 &tcp_force_detection, 0,
153 "Do we force detection even if the INP has it off?");
154 int32_t tcp_sad_limit = 10000;
155 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, limit,
156 CTLFLAG_RW,
157 &tcp_sad_limit, 10000,
158 "If SaD is enabled, what is the limit to sendmap entries (0 = unlimited)?");
159 int32_t tcp_sack_to_ack_thresh = 700; /* 70 % */
160 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sack_to_ack_thresh,
161 CTLFLAG_RW,
162 &tcp_sack_to_ack_thresh, 700,
163 "Percentage of sacks to acks we must see above (10.1 percent is 101)?");
164 int32_t tcp_sack_to_move_thresh = 600; /* 60 % */
165 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, move_thresh,
166 CTLFLAG_RW,
167 &tcp_sack_to_move_thresh, 600,
168 "Percentage of sack moves we must see above (10.1 percent is 101)");
169 int32_t tcp_restoral_thresh = 450; /* 45 % (sack:2:ack -25%) (mv:ratio -15%) **/
170 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, restore_thresh,
171 CTLFLAG_RW,
172 &tcp_restoral_thresh, 450,
173 "Percentage of sack to ack percentage we must see below to restore(10.1 percent is 101)");
174 int32_t tcp_sad_decay_val = 800;
175 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, decay_per,
176 CTLFLAG_RW,
177 &tcp_sad_decay_val, 800,
178 "The decay percentage (10.1 percent equals 101 )");
179 int32_t tcp_map_minimum = 500;
180 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, nummaps,
181 CTLFLAG_RW,
182 &tcp_map_minimum, 500,
183 "Number of Map enteries before we start detection");
184 int32_t tcp_sad_pacing_interval = 2000;
185 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_pacing_int,
186 CTLFLAG_RW,
187 &tcp_sad_pacing_interval, 2000,
188 "What is the minimum pacing interval for a classified attacker?");
189
190 int32_t tcp_sad_low_pps = 100;
191 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_low_pps,
192 CTLFLAG_RW,
193 &tcp_sad_low_pps, 100,
194 "What is the input pps that below which we do not decay?");
195 #endif
196 VNET_DEFINE(uint32_t, tcp_ack_war_time_window) = 1000;
197 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_timewindow,
198 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_ack_war_time_window), 0,
199 "Time interval in ms used to limit the number (ack_war_cnt) of challenge ACKs sent per TCP connection");
200 VNET_DEFINE(uint32_t, tcp_ack_war_cnt) = 5;
201 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_cnt, CTLFLAG_VNET | CTLFLAG_RW,
202 &VNET_NAME(tcp_ack_war_cnt), 0,
203 "Maximum number of challenge ACKs sent per TCP connection during the time interval (ack_war_timewindow)");
204
205 struct rwlock tcp_function_lock;
206
207 static int
sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)208 sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
209 {
210 int error, new;
211
212 new = V_tcp_mssdflt;
213 error = sysctl_handle_int(oidp, &new, 0, req);
214 if (error == 0 && req->newptr) {
215 if (new < TCP_MINMSS)
216 error = EINVAL;
217 else
218 V_tcp_mssdflt = new;
219 }
220 return (error);
221 }
222
223 SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
224 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
225 &VNET_NAME(tcp_mssdflt), 0, &sysctl_net_inet_tcp_mss_check, "I",
226 "Default TCP Maximum Segment Size");
227
228 #ifdef INET6
229 static int
sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)230 sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
231 {
232 int error, new;
233
234 new = V_tcp_v6mssdflt;
235 error = sysctl_handle_int(oidp, &new, 0, req);
236 if (error == 0 && req->newptr) {
237 if (new < TCP_MINMSS)
238 error = EINVAL;
239 else
240 V_tcp_v6mssdflt = new;
241 }
242 return (error);
243 }
244
245 SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
246 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
247 &VNET_NAME(tcp_v6mssdflt), 0, &sysctl_net_inet_tcp_mss_v6_check, "I",
248 "Default TCP Maximum Segment Size for IPv6");
249 #endif /* INET6 */
250
251 /*
252 * Minimum MSS we accept and use. This prevents DoS attacks where
253 * we are forced to a ridiculous low MSS like 20 and send hundreds
254 * of packets instead of one. The effect scales with the available
255 * bandwidth and quickly saturates the CPU and network interface
256 * with packet generation and sending. Set to zero to disable MINMSS
257 * checking. This setting prevents us from sending too small packets.
258 */
259 VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
260 SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET | CTLFLAG_RW,
261 &VNET_NAME(tcp_minmss), 0,
262 "Minimum TCP Maximum Segment Size");
263
264 VNET_DEFINE(int, tcp_do_rfc1323) = 1;
265 SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET | CTLFLAG_RW,
266 &VNET_NAME(tcp_do_rfc1323), 0,
267 "Enable rfc1323 (high performance TCP) extensions");
268
269 /*
270 * As of June 2021, several TCP stacks violate RFC 7323 from September 2014.
271 * Some stacks negotiate TS, but never send them after connection setup. Some
272 * stacks negotiate TS, but don't send them when sending keep-alive segments.
273 * These include modern widely deployed TCP stacks.
274 * Therefore tolerating violations for now...
275 */
276 VNET_DEFINE(int, tcp_tolerate_missing_ts) = 1;
277 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tolerate_missing_ts, CTLFLAG_VNET | CTLFLAG_RW,
278 &VNET_NAME(tcp_tolerate_missing_ts), 0,
279 "Tolerate missing TCP timestamps");
280
281 VNET_DEFINE(int, tcp_ts_offset_per_conn) = 1;
282 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ts_offset_per_conn, CTLFLAG_VNET | CTLFLAG_RW,
283 &VNET_NAME(tcp_ts_offset_per_conn), 0,
284 "Initialize TCP timestamps per connection instead of per host pair");
285
286 /* How many connections are pacing */
287 static volatile uint32_t number_of_tcp_connections_pacing = 0;
288 static uint32_t shadow_num_connections = 0;
289 static counter_u64_t tcp_pacing_failures;
290
291 static int tcp_pacing_limit = 10000;
292 SYSCTL_INT(_net_inet_tcp, OID_AUTO, pacing_limit, CTLFLAG_RW,
293 &tcp_pacing_limit, 1000,
294 "If the TCP stack does pacing, is there a limit (-1 = no, 0 = no pacing N = number of connections)");
295
296 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pacing_count, CTLFLAG_RD,
297 &shadow_num_connections, 0, "Number of TCP connections being paced");
298
299 SYSCTL_COUNTER_U64(_net_inet_tcp, OID_AUTO, pacing_failures, CTLFLAG_RD,
300 &tcp_pacing_failures, "Number of times we failed to enable pacing to avoid exceeding the limit");
301
302 static int tcp_log_debug = 0;
303 SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
304 &tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
305
306 /*
307 * Target size of TCP PCB hash tables. Must be a power of two.
308 *
309 * Note that this can be overridden by the kernel environment
310 * variable net.inet.tcp.tcbhashsize
311 */
312 #ifndef TCBHASHSIZE
313 #define TCBHASHSIZE 0
314 #endif
315 static int tcp_tcbhashsize = TCBHASHSIZE;
316 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN,
317 &tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
318
319 static int do_tcpdrain = 1;
320 SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
321 "Enable tcp_drain routine for extra help when low on mbufs");
322
323 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET | CTLFLAG_RD,
324 &VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");
325
326 VNET_DEFINE_STATIC(int, icmp_may_rst) = 1;
327 #define V_icmp_may_rst VNET(icmp_may_rst)
328 SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET | CTLFLAG_RW,
329 &VNET_NAME(icmp_may_rst), 0,
330 "Certain ICMP unreachable messages may abort connections in SYN_SENT");
331
332 VNET_DEFINE_STATIC(int, tcp_isn_reseed_interval) = 0;
333 #define V_tcp_isn_reseed_interval VNET(tcp_isn_reseed_interval)
334 SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET | CTLFLAG_RW,
335 &VNET_NAME(tcp_isn_reseed_interval), 0,
336 "Seconds between reseeding of ISN secret");
337
338 static int tcp_soreceive_stream;
339 SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
340 &tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
341
342 VNET_DEFINE(uma_zone_t, sack_hole_zone);
343 #define V_sack_hole_zone VNET(sack_hole_zone)
344 VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0; /* unlimited */
345 static int
sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)346 sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
347 {
348 int error;
349 uint32_t new;
350
351 new = V_tcp_map_entries_limit;
352 error = sysctl_handle_int(oidp, &new, 0, req);
353 if (error == 0 && req->newptr) {
354 /* only allow "0" and value > minimum */
355 if (new > 0 && new < TCP_MIN_MAP_ENTRIES_LIMIT)
356 error = EINVAL;
357 else
358 V_tcp_map_entries_limit = new;
359 }
360 return (error);
361 }
362 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, map_limit,
363 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
364 &VNET_NAME(tcp_map_entries_limit), 0,
365 &sysctl_net_inet_tcp_map_limit_check, "IU",
366 "Total sendmap entries limit");
367
368 VNET_DEFINE(uint32_t, tcp_map_split_limit) = 0; /* unlimited */
369 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, split_limit, CTLFLAG_VNET | CTLFLAG_RW,
370 &VNET_NAME(tcp_map_split_limit), 0,
371 "Total sendmap split entries limit");
372
373 #ifdef TCP_HHOOK
374 VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
375 #endif
376
377 #define TS_OFFSET_SECRET_LENGTH SIPHASH_KEY_LENGTH
378 VNET_DEFINE_STATIC(u_char, ts_offset_secret[TS_OFFSET_SECRET_LENGTH]);
379 #define V_ts_offset_secret VNET(ts_offset_secret)
380
381 static int tcp_default_fb_init(struct tcpcb *tp, void **ptr);
382 static void tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged);
383 static int tcp_default_handoff_ok(struct tcpcb *tp);
384 static struct inpcb *tcp_notify(struct inpcb *, int);
385 static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
386 static struct inpcb *tcp_mtudisc(struct inpcb *, int);
387 static struct inpcb *tcp_drop_syn_sent(struct inpcb *, int);
388 static char * tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
389 const void *ip4hdr, const void *ip6hdr);
390 static void tcp_default_switch_failed(struct tcpcb *tp);
391 static ipproto_ctlinput_t tcp_ctlinput;
392 static udp_tun_icmp_t tcp_ctlinput_viaudp;
393
394 static struct tcp_function_block tcp_def_funcblk = {
395 .tfb_tcp_block_name = "freebsd",
396 .tfb_tcp_output = tcp_default_output,
397 .tfb_tcp_do_segment = tcp_do_segment,
398 .tfb_tcp_ctloutput = tcp_default_ctloutput,
399 .tfb_tcp_handoff_ok = tcp_default_handoff_ok,
400 .tfb_tcp_fb_init = tcp_default_fb_init,
401 .tfb_tcp_fb_fini = tcp_default_fb_fini,
402 .tfb_switch_failed = tcp_default_switch_failed,
403 .tfb_flags = TCP_FUNC_DEFAULT_OK,
404 };
405
406 static int tcp_fb_cnt = 0;
407 struct tcp_funchead t_functions;
408 VNET_DEFINE_STATIC(struct tcp_function_block *, tcp_func_set_ptr) = &tcp_def_funcblk;
409 #define V_tcp_func_set_ptr VNET(tcp_func_set_ptr)
410
411 void
tcp_record_dsack(struct tcpcb * tp,tcp_seq start,tcp_seq end,int tlp)412 tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
413 {
414 TCPSTAT_INC(tcps_dsack_count);
415 tp->t_dsack_pack++;
416 if (tlp == 0) {
417 if (SEQ_GT(end, start)) {
418 tp->t_dsack_bytes += (end - start);
419 TCPSTAT_ADD(tcps_dsack_bytes, (end - start));
420 } else {
421 tp->t_dsack_tlp_bytes += (start - end);
422 TCPSTAT_ADD(tcps_dsack_bytes, (start - end));
423 }
424 } else {
425 if (SEQ_GT(end, start)) {
426 tp->t_dsack_bytes += (end - start);
427 TCPSTAT_ADD(tcps_dsack_tlp_bytes, (end - start));
428 } else {
429 tp->t_dsack_tlp_bytes += (start - end);
430 TCPSTAT_ADD(tcps_dsack_tlp_bytes, (start - end));
431 }
432 }
433 }
434
435 static struct tcp_function_block *
find_tcp_functions_locked(struct tcp_function_set * fs)436 find_tcp_functions_locked(struct tcp_function_set *fs)
437 {
438 struct tcp_function *f;
439 struct tcp_function_block *blk = NULL;
440
441 rw_assert(&tcp_function_lock, RA_LOCKED);
442 TAILQ_FOREACH(f, &t_functions, tf_next) {
443 if (strcmp(f->tf_name, fs->function_set_name) == 0) {
444 blk = f->tf_fb;
445 break;
446 }
447 }
448 return (blk);
449 }
450
451 static struct tcp_function_block *
find_tcp_fb_locked(struct tcp_function_block * blk,struct tcp_function ** s)452 find_tcp_fb_locked(struct tcp_function_block *blk, struct tcp_function **s)
453 {
454 struct tcp_function_block *rblk = NULL;
455 struct tcp_function *f;
456
457 rw_assert(&tcp_function_lock, RA_LOCKED);
458 TAILQ_FOREACH(f, &t_functions, tf_next) {
459 if (f->tf_fb == blk) {
460 rblk = blk;
461 if (s) {
462 *s = f;
463 }
464 break;
465 }
466 }
467 return (rblk);
468 }
469
470 struct tcp_function_block *
find_and_ref_tcp_functions(struct tcp_function_set * fs)471 find_and_ref_tcp_functions(struct tcp_function_set *fs)
472 {
473 struct tcp_function_block *blk;
474
475 rw_rlock(&tcp_function_lock);
476 blk = find_tcp_functions_locked(fs);
477 if (blk)
478 refcount_acquire(&blk->tfb_refcnt);
479 rw_runlock(&tcp_function_lock);
480 return (blk);
481 }
482
483 struct tcp_function_block *
find_and_ref_tcp_fb(struct tcp_function_block * blk)484 find_and_ref_tcp_fb(struct tcp_function_block *blk)
485 {
486 struct tcp_function_block *rblk;
487
488 rw_rlock(&tcp_function_lock);
489 rblk = find_tcp_fb_locked(blk, NULL);
490 if (rblk)
491 refcount_acquire(&rblk->tfb_refcnt);
492 rw_runlock(&tcp_function_lock);
493 return (rblk);
494 }
495
496 /* Find a matching alias for the given tcp_function_block. */
497 int
find_tcp_function_alias(struct tcp_function_block * blk,struct tcp_function_set * fs)498 find_tcp_function_alias(struct tcp_function_block *blk,
499 struct tcp_function_set *fs)
500 {
501 struct tcp_function *f;
502 int found;
503
504 found = 0;
505 rw_rlock(&tcp_function_lock);
506 TAILQ_FOREACH(f, &t_functions, tf_next) {
507 if ((f->tf_fb == blk) &&
508 (strncmp(f->tf_name, blk->tfb_tcp_block_name,
509 TCP_FUNCTION_NAME_LEN_MAX) != 0)) {
510 /* Matching function block with different name. */
511 strncpy(fs->function_set_name, f->tf_name,
512 TCP_FUNCTION_NAME_LEN_MAX);
513 found = 1;
514 break;
515 }
516 }
517 /* Null terminate the string appropriately. */
518 if (found) {
519 fs->function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
520 } else {
521 fs->function_set_name[0] = '\0';
522 }
523 rw_runlock(&tcp_function_lock);
524 return (found);
525 }
526
527 static struct tcp_function_block *
find_and_ref_tcp_default_fb(void)528 find_and_ref_tcp_default_fb(void)
529 {
530 struct tcp_function_block *rblk;
531
532 rw_rlock(&tcp_function_lock);
533 rblk = V_tcp_func_set_ptr;
534 refcount_acquire(&rblk->tfb_refcnt);
535 rw_runlock(&tcp_function_lock);
536 return (rblk);
537 }
538
539 void
tcp_switch_back_to_default(struct tcpcb * tp)540 tcp_switch_back_to_default(struct tcpcb *tp)
541 {
542 struct tcp_function_block *tfb;
543 void *ptr = NULL;
544
545 KASSERT(tp->t_fb != &tcp_def_funcblk,
546 ("%s: called by the built-in default stack", __func__));
547
548 if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
549 tp->t_fb->tfb_tcp_timer_stop_all(tp);
550
551 /*
552 * Now, we'll find a new function block to use.
553 * Start by trying the current user-selected
554 * default, unless this stack is the user-selected
555 * default.
556 */
557 tfb = find_and_ref_tcp_default_fb();
558 if (tfb == tp->t_fb) {
559 refcount_release(&tfb->tfb_refcnt);
560 tfb = NULL;
561 }
562 /* Does the stack accept this connection? */
563 if (tfb != NULL && (*tfb->tfb_tcp_handoff_ok)(tp)) {
564 refcount_release(&tfb->tfb_refcnt);
565 tfb = NULL;
566 }
567 /* Try to use that stack. */
568 if (tfb != NULL) {
569 /* Initialize the new stack. If it succeeds, we are done. */
570 if (tfb->tfb_tcp_fb_init == NULL ||
571 (*tfb->tfb_tcp_fb_init)(tp, &ptr) == 0) {
572 /* Release the old stack */
573 if (tp->t_fb->tfb_tcp_fb_fini != NULL)
574 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
575 refcount_release(&tp->t_fb->tfb_refcnt);
576 /* Now set in all the pointers */
577 tp->t_fb = tfb;
578 tp->t_fb_ptr = ptr;
579 return;
580 }
581 /*
582 * Initialization failed. Release the reference count on
583 * the looked up default stack.
584 */
585 refcount_release(&tfb->tfb_refcnt);
586 }
587
588 /*
589 * If that wasn't feasible, use the built-in default
590 * stack which is not allowed to reject anyone.
591 */
592 tfb = find_and_ref_tcp_fb(&tcp_def_funcblk);
593 if (tfb == NULL) {
594 /* there always should be a default */
595 panic("Can't refer to tcp_def_funcblk");
596 }
597 if ((*tfb->tfb_tcp_handoff_ok)(tp)) {
598 /* The default stack cannot say no */
599 panic("Default stack rejects a new session?");
600 }
601 if (tfb->tfb_tcp_fb_init != NULL &&
602 (*tfb->tfb_tcp_fb_init)(tp, &ptr)) {
603 /* The default stack cannot fail */
604 panic("Default stack initialization failed");
605 }
606 /* Now release the old stack */
607 if (tp->t_fb->tfb_tcp_fb_fini != NULL)
608 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
609 refcount_release(&tp->t_fb->tfb_refcnt);
610 /* And set in the pointers to the new */
611 tp->t_fb = tfb;
612 tp->t_fb_ptr = ptr;
613 }
614
615 static bool
tcp_recv_udp_tunneled_packet(struct mbuf * m,int off,struct inpcb * inp,const struct sockaddr * sa,void * ctx)616 tcp_recv_udp_tunneled_packet(struct mbuf *m, int off, struct inpcb *inp,
617 const struct sockaddr *sa, void *ctx)
618 {
619 struct ip *iph;
620 #ifdef INET6
621 struct ip6_hdr *ip6;
622 #endif
623 struct udphdr *uh;
624 struct tcphdr *th;
625 int thlen;
626 uint16_t port;
627
628 TCPSTAT_INC(tcps_tunneled_pkts);
629 if ((m->m_flags & M_PKTHDR) == 0) {
630 /* Can't handle one that is not a pkt hdr */
631 TCPSTAT_INC(tcps_tunneled_errs);
632 goto out;
633 }
634 thlen = sizeof(struct tcphdr);
635 if (m->m_len < off + sizeof(struct udphdr) + thlen &&
636 (m = m_pullup(m, off + sizeof(struct udphdr) + thlen)) == NULL) {
637 TCPSTAT_INC(tcps_tunneled_errs);
638 goto out;
639 }
640 iph = mtod(m, struct ip *);
641 uh = (struct udphdr *)((caddr_t)iph + off);
642 th = (struct tcphdr *)(uh + 1);
643 thlen = th->th_off << 2;
644 if (m->m_len < off + sizeof(struct udphdr) + thlen) {
645 m = m_pullup(m, off + sizeof(struct udphdr) + thlen);
646 if (m == NULL) {
647 TCPSTAT_INC(tcps_tunneled_errs);
648 goto out;
649 } else {
650 iph = mtod(m, struct ip *);
651 uh = (struct udphdr *)((caddr_t)iph + off);
652 th = (struct tcphdr *)(uh + 1);
653 }
654 }
655 m->m_pkthdr.tcp_tun_port = port = uh->uh_sport;
656 bcopy(th, uh, m->m_len - off);
657 m->m_len -= sizeof(struct udphdr);
658 m->m_pkthdr.len -= sizeof(struct udphdr);
659 /*
660 * We use the same algorithm for
661 * both UDP and TCP for c-sum. So
662 * the code in tcp_input will skip
663 * the checksum. So we do nothing
664 * with the flag (m->m_pkthdr.csum_flags).
665 */
666 switch (iph->ip_v) {
667 #ifdef INET
668 case IPVERSION:
669 iph->ip_len = htons(ntohs(iph->ip_len) - sizeof(struct udphdr));
670 tcp_input_with_port(&m, &off, IPPROTO_TCP, port);
671 break;
672 #endif
673 #ifdef INET6
674 case IPV6_VERSION >> 4:
675 ip6 = mtod(m, struct ip6_hdr *);
676 ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - sizeof(struct udphdr));
677 tcp6_input_with_port(&m, &off, IPPROTO_TCP, port);
678 break;
679 #endif
680 default:
681 goto out;
682 break;
683 }
684 return (true);
685 out:
686 m_freem(m);
687
688 return (true);
689 }
690
691 static int
sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)692 sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)
693 {
694 int error = ENOENT;
695 struct tcp_function_set fs;
696 struct tcp_function_block *blk;
697
698 memset(&fs, 0, sizeof(fs));
699 rw_rlock(&tcp_function_lock);
700 blk = find_tcp_fb_locked(V_tcp_func_set_ptr, NULL);
701 if (blk) {
702 /* Found him */
703 strcpy(fs.function_set_name, blk->tfb_tcp_block_name);
704 fs.pcbcnt = blk->tfb_refcnt;
705 }
706 rw_runlock(&tcp_function_lock);
707 error = sysctl_handle_string(oidp, fs.function_set_name,
708 sizeof(fs.function_set_name), req);
709
710 /* Check for error or no change */
711 if (error != 0 || req->newptr == NULL)
712 return (error);
713
714 rw_wlock(&tcp_function_lock);
715 blk = find_tcp_functions_locked(&fs);
716 if ((blk == NULL) ||
717 (blk->tfb_flags & TCP_FUNC_BEING_REMOVED)) {
718 error = ENOENT;
719 goto done;
720 }
721 if ((blk->tfb_flags & TCP_FUNC_DEFAULT_OK) == 0) {
722 error = EINVAL;
723 goto done;
724 }
725 V_tcp_func_set_ptr = blk;
726 done:
727 rw_wunlock(&tcp_function_lock);
728 return (error);
729 }
730
731 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_default,
732 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
733 NULL, 0, sysctl_net_inet_default_tcp_functions, "A",
734 "Set/get the default TCP functions");
735
736 static int
sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)737 sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)
738 {
739 int error, cnt, linesz;
740 struct tcp_function *f;
741 char *buffer, *cp;
742 size_t bufsz, outsz;
743 bool alias;
744
745 cnt = 0;
746 rw_rlock(&tcp_function_lock);
747 TAILQ_FOREACH(f, &t_functions, tf_next) {
748 cnt++;
749 }
750 rw_runlock(&tcp_function_lock);
751
752 bufsz = (cnt+2) * ((TCP_FUNCTION_NAME_LEN_MAX * 2) + 13) + 1;
753 buffer = malloc(bufsz, M_TEMP, M_WAITOK);
754
755 error = 0;
756 cp = buffer;
757
758 linesz = snprintf(cp, bufsz, "\n%-32s%c %-32s %s\n", "Stack", 'D',
759 "Alias", "PCB count");
760 cp += linesz;
761 bufsz -= linesz;
762 outsz = linesz;
763
764 rw_rlock(&tcp_function_lock);
765 TAILQ_FOREACH(f, &t_functions, tf_next) {
766 alias = (f->tf_name != f->tf_fb->tfb_tcp_block_name);
767 linesz = snprintf(cp, bufsz, "%-32s%c %-32s %u\n",
768 f->tf_fb->tfb_tcp_block_name,
769 (f->tf_fb == V_tcp_func_set_ptr) ? '*' : ' ',
770 alias ? f->tf_name : "-",
771 f->tf_fb->tfb_refcnt);
772 if (linesz >= bufsz) {
773 error = EOVERFLOW;
774 break;
775 }
776 cp += linesz;
777 bufsz -= linesz;
778 outsz += linesz;
779 }
780 rw_runlock(&tcp_function_lock);
781 if (error == 0)
782 error = sysctl_handle_string(oidp, buffer, outsz + 1, req);
783 free(buffer, M_TEMP);
784 return (error);
785 }
786
787 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available,
788 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
789 NULL, 0, sysctl_net_inet_list_available, "A",
790 "list available TCP Function sets");
791
792 VNET_DEFINE(int, tcp_udp_tunneling_port) = TCP_TUNNELING_PORT_DEFAULT;
793
794 #ifdef INET
795 VNET_DEFINE(struct socket *, udp4_tun_socket) = NULL;
796 #define V_udp4_tun_socket VNET(udp4_tun_socket)
797 #endif
798 #ifdef INET6
799 VNET_DEFINE(struct socket *, udp6_tun_socket) = NULL;
800 #define V_udp6_tun_socket VNET(udp6_tun_socket)
801 #endif
802
803 static struct sx tcpoudp_lock;
804
805 static void
tcp_over_udp_stop(void)806 tcp_over_udp_stop(void)
807 {
808
809 sx_assert(&tcpoudp_lock, SA_XLOCKED);
810
811 #ifdef INET
812 if (V_udp4_tun_socket != NULL) {
813 soclose(V_udp4_tun_socket);
814 V_udp4_tun_socket = NULL;
815 }
816 #endif
817 #ifdef INET6
818 if (V_udp6_tun_socket != NULL) {
819 soclose(V_udp6_tun_socket);
820 V_udp6_tun_socket = NULL;
821 }
822 #endif
823 }
824
825 static int
tcp_over_udp_start(void)826 tcp_over_udp_start(void)
827 {
828 uint16_t port;
829 int ret;
830 #ifdef INET
831 struct sockaddr_in sin;
832 #endif
833 #ifdef INET6
834 struct sockaddr_in6 sin6;
835 #endif
836
837 sx_assert(&tcpoudp_lock, SA_XLOCKED);
838
839 port = V_tcp_udp_tunneling_port;
840 if (ntohs(port) == 0) {
841 /* Must have a port set */
842 return (EINVAL);
843 }
844 #ifdef INET
845 if (V_udp4_tun_socket != NULL) {
846 /* Already running -- must stop first */
847 return (EALREADY);
848 }
849 #endif
850 #ifdef INET6
851 if (V_udp6_tun_socket != NULL) {
852 /* Already running -- must stop first */
853 return (EALREADY);
854 }
855 #endif
856 #ifdef INET
857 if ((ret = socreate(PF_INET, &V_udp4_tun_socket,
858 SOCK_DGRAM, IPPROTO_UDP,
859 curthread->td_ucred, curthread))) {
860 tcp_over_udp_stop();
861 return (ret);
862 }
863 /* Call the special UDP hook. */
864 if ((ret = udp_set_kernel_tunneling(V_udp4_tun_socket,
865 tcp_recv_udp_tunneled_packet,
866 tcp_ctlinput_viaudp,
867 NULL))) {
868 tcp_over_udp_stop();
869 return (ret);
870 }
871 /* Ok, we have a socket, bind it to the port. */
872 memset(&sin, 0, sizeof(struct sockaddr_in));
873 sin.sin_len = sizeof(struct sockaddr_in);
874 sin.sin_family = AF_INET;
875 sin.sin_port = htons(port);
876 if ((ret = sobind(V_udp4_tun_socket,
877 (struct sockaddr *)&sin, curthread))) {
878 tcp_over_udp_stop();
879 return (ret);
880 }
881 #endif
882 #ifdef INET6
883 if ((ret = socreate(PF_INET6, &V_udp6_tun_socket,
884 SOCK_DGRAM, IPPROTO_UDP,
885 curthread->td_ucred, curthread))) {
886 tcp_over_udp_stop();
887 return (ret);
888 }
889 /* Call the special UDP hook. */
890 if ((ret = udp_set_kernel_tunneling(V_udp6_tun_socket,
891 tcp_recv_udp_tunneled_packet,
892 tcp6_ctlinput_viaudp,
893 NULL))) {
894 tcp_over_udp_stop();
895 return (ret);
896 }
897 /* Ok, we have a socket, bind it to the port. */
898 memset(&sin6, 0, sizeof(struct sockaddr_in6));
899 sin6.sin6_len = sizeof(struct sockaddr_in6);
900 sin6.sin6_family = AF_INET6;
901 sin6.sin6_port = htons(port);
902 if ((ret = sobind(V_udp6_tun_socket,
903 (struct sockaddr *)&sin6, curthread))) {
904 tcp_over_udp_stop();
905 return (ret);
906 }
907 #endif
908 return (0);
909 }
910
911 static int
sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)912 sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)
913 {
914 int error;
915 uint32_t old, new;
916
917 old = V_tcp_udp_tunneling_port;
918 new = old;
919 error = sysctl_handle_int(oidp, &new, 0, req);
920 if ((error == 0) &&
921 (req->newptr != NULL)) {
922 if ((new < TCP_TUNNELING_PORT_MIN) ||
923 (new > TCP_TUNNELING_PORT_MAX)) {
924 error = EINVAL;
925 } else {
926 sx_xlock(&tcpoudp_lock);
927 V_tcp_udp_tunneling_port = new;
928 if (old != 0) {
929 tcp_over_udp_stop();
930 }
931 if (new != 0) {
932 error = tcp_over_udp_start();
933 if (error != 0) {
934 V_tcp_udp_tunneling_port = 0;
935 }
936 }
937 sx_xunlock(&tcpoudp_lock);
938 }
939 }
940 return (error);
941 }
942
943 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_port,
944 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
945 &VNET_NAME(tcp_udp_tunneling_port),
946 0, &sysctl_net_inet_tcp_udp_tunneling_port_check, "IU",
947 "Tunneling port for tcp over udp");
948
949 VNET_DEFINE(int, tcp_udp_tunneling_overhead) = TCP_TUNNELING_OVERHEAD_DEFAULT;
950
951 static int
sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)952 sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)
953 {
954 int error, new;
955
956 new = V_tcp_udp_tunneling_overhead;
957 error = sysctl_handle_int(oidp, &new, 0, req);
958 if (error == 0 && req->newptr) {
959 if ((new < TCP_TUNNELING_OVERHEAD_MIN) ||
960 (new > TCP_TUNNELING_OVERHEAD_MAX))
961 error = EINVAL;
962 else
963 V_tcp_udp_tunneling_overhead = new;
964 }
965 return (error);
966 }
967
968 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_overhead,
969 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
970 &VNET_NAME(tcp_udp_tunneling_overhead),
971 0, &sysctl_net_inet_tcp_udp_tunneling_overhead_check, "IU",
972 "MSS reduction when using tcp over udp");
973
974 /*
975 * Exports one (struct tcp_function_info) for each alias/name.
976 */
977 static int
sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)978 sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)
979 {
980 int cnt, error;
981 struct tcp_function *f;
982 struct tcp_function_info tfi;
983
984 /*
985 * We don't allow writes.
986 */
987 if (req->newptr != NULL)
988 return (EINVAL);
989
990 /*
991 * Wire the old buffer so we can directly copy the functions to
992 * user space without dropping the lock.
993 */
994 if (req->oldptr != NULL) {
995 error = sysctl_wire_old_buffer(req, 0);
996 if (error)
997 return (error);
998 }
999
1000 /*
1001 * Walk the list and copy out matching entries. If INVARIANTS
1002 * is compiled in, also walk the list to verify the length of
1003 * the list matches what we have recorded.
1004 */
1005 rw_rlock(&tcp_function_lock);
1006
1007 cnt = 0;
1008 #ifndef INVARIANTS
1009 if (req->oldptr == NULL) {
1010 cnt = tcp_fb_cnt;
1011 goto skip_loop;
1012 }
1013 #endif
1014 TAILQ_FOREACH(f, &t_functions, tf_next) {
1015 #ifdef INVARIANTS
1016 cnt++;
1017 #endif
1018 if (req->oldptr != NULL) {
1019 bzero(&tfi, sizeof(tfi));
1020 tfi.tfi_refcnt = f->tf_fb->tfb_refcnt;
1021 tfi.tfi_id = f->tf_fb->tfb_id;
1022 (void)strlcpy(tfi.tfi_alias, f->tf_name,
1023 sizeof(tfi.tfi_alias));
1024 (void)strlcpy(tfi.tfi_name,
1025 f->tf_fb->tfb_tcp_block_name, sizeof(tfi.tfi_name));
1026 error = SYSCTL_OUT(req, &tfi, sizeof(tfi));
1027 /*
1028 * Don't stop on error, as that is the
1029 * mechanism we use to accumulate length
1030 * information if the buffer was too short.
1031 */
1032 }
1033 }
1034 KASSERT(cnt == tcp_fb_cnt,
1035 ("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt));
1036 #ifndef INVARIANTS
1037 skip_loop:
1038 #endif
1039 rw_runlock(&tcp_function_lock);
1040 if (req->oldptr == NULL)
1041 error = SYSCTL_OUT(req, NULL,
1042 (cnt + 1) * sizeof(struct tcp_function_info));
1043
1044 return (error);
1045 }
1046
1047 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info,
1048 CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE,
1049 NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info",
1050 "List TCP function block name-to-ID mappings");
1051
1052 /*
1053 * tfb_tcp_handoff_ok() function for the default stack.
1054 * Note that we'll basically try to take all comers.
1055 */
1056 static int
tcp_default_handoff_ok(struct tcpcb * tp)1057 tcp_default_handoff_ok(struct tcpcb *tp)
1058 {
1059
1060 return (0);
1061 }
1062
1063 /*
1064 * tfb_tcp_fb_init() function for the default stack.
1065 *
1066 * This handles making sure we have appropriate timers set if you are
1067 * transitioning a socket that has some amount of setup done.
1068 *
1069 * The init() fuction from the default can *never* return non-zero i.e.
1070 * it is required to always succeed since it is the stack of last resort!
1071 */
1072 static int
tcp_default_fb_init(struct tcpcb * tp,void ** ptr)1073 tcp_default_fb_init(struct tcpcb *tp, void **ptr)
1074 {
1075 struct socket *so = tptosocket(tp);
1076 int rexmt;
1077
1078 INP_WLOCK_ASSERT(tptoinpcb(tp));
1079 /* We don't use the pointer */
1080 *ptr = NULL;
1081
1082 KASSERT(tp->t_state < TCPS_TIME_WAIT,
1083 ("%s: connection %p in unexpected state %d", __func__, tp,
1084 tp->t_state));
1085
1086 /* Make sure we get no interesting mbuf queuing behavior */
1087 /* All mbuf queue/ack compress flags should be off */
1088 tcp_lro_features_off(tp);
1089
1090 /* Cancel the GP measurement in progress */
1091 tp->t_flags &= ~TF_GPUTINPROG;
1092 /* Validate the timers are not in usec, if they are convert */
1093 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
1094 if ((tp->t_state == TCPS_SYN_SENT) ||
1095 (tp->t_state == TCPS_SYN_RECEIVED))
1096 rexmt = tcp_rexmit_initial * tcp_backoff[tp->t_rxtshift];
1097 else
1098 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1099 if (tp->t_rxtshift == 0)
1100 tp->t_rxtcur = rexmt;
1101 else
1102 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX);
1103
1104 /*
1105 * Nothing to do for ESTABLISHED or LISTEN states. And, we don't
1106 * know what to do for unexpected states (which includes TIME_WAIT).
1107 */
1108 if (tp->t_state <= TCPS_LISTEN || tp->t_state >= TCPS_TIME_WAIT)
1109 return (0);
1110
1111 /*
1112 * Make sure some kind of transmission timer is set if there is
1113 * outstanding data.
1114 */
1115 if ((!TCPS_HAVEESTABLISHED(tp->t_state) || sbavail(&so->so_snd) ||
1116 tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) ||
1117 tcp_timer_active(tp, TT_PERSIST))) {
1118 /*
1119 * If the session has established and it looks like it should
1120 * be in the persist state, set the persist timer. Otherwise,
1121 * set the retransmit timer.
1122 */
1123 if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 &&
1124 (int32_t)(tp->snd_nxt - tp->snd_una) <
1125 (int32_t)sbavail(&so->so_snd))
1126 tcp_setpersist(tp);
1127 else
1128 tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
1129 }
1130
1131 /* All non-embryonic sessions get a keepalive timer. */
1132 if (!tcp_timer_active(tp, TT_KEEP))
1133 tcp_timer_activate(tp, TT_KEEP,
1134 TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) :
1135 TP_KEEPINIT(tp));
1136
1137 /*
1138 * Make sure critical variables are initialized
1139 * if transitioning while in Recovery.
1140 */
1141 if IN_FASTRECOVERY(tp->t_flags) {
1142 if (tp->sackhint.recover_fs == 0)
1143 tp->sackhint.recover_fs = max(1,
1144 tp->snd_nxt - tp->snd_una);
1145 }
1146
1147 return (0);
1148 }
1149
1150 /*
1151 * tfb_tcp_fb_fini() function for the default stack.
1152 *
1153 * This changes state as necessary (or prudent) to prepare for another stack
1154 * to assume responsibility for the connection.
1155 */
1156 static void
tcp_default_fb_fini(struct tcpcb * tp,int tcb_is_purged)1157 tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged)
1158 {
1159
1160 INP_WLOCK_ASSERT(tptoinpcb(tp));
1161
1162 #ifdef TCP_BLACKBOX
1163 tcp_log_flowend(tp);
1164 #endif
1165 tp->t_acktime = 0;
1166 return;
1167 }
1168
1169 MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
1170 MALLOC_DEFINE(M_TCPFUNCTIONS, "tcpfunc", "TCP function set memory");
1171
1172 static struct mtx isn_mtx;
1173
1174 #define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
1175 #define ISN_LOCK() mtx_lock(&isn_mtx)
1176 #define ISN_UNLOCK() mtx_unlock(&isn_mtx)
1177
1178 INPCBSTORAGE_DEFINE(tcpcbstor, tcpcb, "tcpinp", "tcp_inpcb", "tcp", "tcphash");
1179
1180 /*
1181 * Take a value and get the next power of 2 that doesn't overflow.
1182 * Used to size the tcp_inpcb hash buckets.
1183 */
1184 static int
maketcp_hashsize(int size)1185 maketcp_hashsize(int size)
1186 {
1187 int hashsize;
1188
1189 /*
1190 * auto tune.
1191 * get the next power of 2 higher than maxsockets.
1192 */
1193 hashsize = 1 << fls(size);
1194 /* catch overflow, and just go one power of 2 smaller */
1195 if (hashsize < size) {
1196 hashsize = 1 << (fls(size) - 1);
1197 }
1198 return (hashsize);
1199 }
1200
1201 static volatile int next_tcp_stack_id = 1;
1202
1203 /*
1204 * Register a TCP function block with the name provided in the names
1205 * array. (Note that this function does NOT automatically register
1206 * blk->tfb_tcp_block_name as a stack name. Therefore, you should
1207 * explicitly include blk->tfb_tcp_block_name in the list of names if
1208 * you wish to register the stack with that name.)
1209 *
1210 * Either all name registrations will succeed or all will fail. If
1211 * a name registration fails, the function will update the num_names
1212 * argument to point to the array index of the name that encountered
1213 * the failure.
1214 *
1215 * Returns 0 on success, or an error code on failure.
1216 */
1217 int
register_tcp_functions_as_names(struct tcp_function_block * blk,int wait,const char * names[],int * num_names)1218 register_tcp_functions_as_names(struct tcp_function_block *blk, int wait,
1219 const char *names[], int *num_names)
1220 {
1221 struct tcp_function *f[TCP_FUNCTION_NAME_NUM_MAX];
1222 struct tcp_function_set fs;
1223 int error, i, num_registered;
1224
1225 KASSERT(names != NULL, ("%s: Called with NULL name list", __func__));
1226 KASSERT(*num_names > 0,
1227 ("%s: Called with non-positive length of name list", __func__));
1228 KASSERT(rw_initialized(&tcp_function_lock),
1229 ("%s: called too early", __func__));
1230
1231 if (*num_names > TCP_FUNCTION_NAME_NUM_MAX) {
1232 /* Too many names. */
1233 *num_names = 0;
1234 return (E2BIG);
1235 }
1236 if ((blk->tfb_tcp_output == NULL) ||
1237 (blk->tfb_tcp_do_segment == NULL) ||
1238 (blk->tfb_tcp_ctloutput == NULL) ||
1239 (blk->tfb_tcp_handoff_ok == NULL) ||
1240 (strlen(blk->tfb_tcp_block_name) == 0)) {
1241 /* These functions are required and a name is needed. */
1242 *num_names = 0;
1243 return (EINVAL);
1244 }
1245
1246 for (i = 0; i < *num_names; i++) {
1247 f[i] = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait);
1248 if (f[i] == NULL) {
1249 while (--i >= 0)
1250 free(f[i], M_TCPFUNCTIONS);
1251 *num_names = 0;
1252 return (ENOMEM);
1253 }
1254 }
1255
1256 num_registered = 0;
1257 rw_wlock(&tcp_function_lock);
1258 if (find_tcp_fb_locked(blk, NULL) != NULL) {
1259 /* A TCP function block can only be registered once. */
1260 error = EALREADY;
1261 goto cleanup;
1262 }
1263 if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1264 error = EINVAL;
1265 goto cleanup;
1266 }
1267 refcount_init(&blk->tfb_refcnt, 0);
1268 blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1);
1269 for (i = 0; i < *num_names; i++) {
1270 (void)strlcpy(fs.function_set_name, names[i],
1271 sizeof(fs.function_set_name));
1272 if (find_tcp_functions_locked(&fs) != NULL) {
1273 /* Duplicate name space not allowed */
1274 error = EALREADY;
1275 goto cleanup;
1276 }
1277 f[i]->tf_fb = blk;
1278 (void)strlcpy(f[i]->tf_name, names[i], sizeof(f[i]->tf_name));
1279 TAILQ_INSERT_TAIL(&t_functions, f[i], tf_next);
1280 tcp_fb_cnt++;
1281 num_registered++;
1282 }
1283 rw_wunlock(&tcp_function_lock);
1284 return (0);
1285
1286 cleanup:
1287 /* Remove the entries just added. */
1288 for (i = 0; i < *num_names; i++) {
1289 if (i < num_registered) {
1290 TAILQ_REMOVE(&t_functions, f[i], tf_next);
1291 tcp_fb_cnt--;
1292 }
1293 f[i]->tf_fb = NULL;
1294 free(f[i], M_TCPFUNCTIONS);
1295 }
1296 rw_wunlock(&tcp_function_lock);
1297 *num_names = num_registered;
1298 return (error);
1299 }
1300
1301 /*
1302 * Register a TCP function block using the name provided in the name
1303 * argument.
1304 *
1305 * Returns 0 on success, or an error code on failure.
1306 */
1307 int
register_tcp_functions_as_name(struct tcp_function_block * blk,const char * name,int wait)1308 register_tcp_functions_as_name(struct tcp_function_block *blk, const char *name,
1309 int wait)
1310 {
1311 const char *name_list[1];
1312 int num_names, rv;
1313
1314 num_names = 1;
1315 if (name != NULL)
1316 name_list[0] = name;
1317 else
1318 name_list[0] = blk->tfb_tcp_block_name;
1319 rv = register_tcp_functions_as_names(blk, wait, name_list, &num_names);
1320 return (rv);
1321 }
1322
1323 /*
1324 * Register a TCP function block using the name defined in
1325 * blk->tfb_tcp_block_name.
1326 *
1327 * Returns 0 on success, or an error code on failure.
1328 */
1329 int
register_tcp_functions(struct tcp_function_block * blk,int wait)1330 register_tcp_functions(struct tcp_function_block *blk, int wait)
1331 {
1332
1333 return (register_tcp_functions_as_name(blk, NULL, wait));
1334 }
1335
1336 /*
1337 * Deregister all names associated with a function block. This
1338 * functionally removes the function block from use within the system.
1339 *
1340 * When called with a true quiesce argument, mark the function block
1341 * as being removed so no more stacks will use it and determine
1342 * whether the removal would succeed.
1343 *
1344 * When called with a false quiesce argument, actually attempt the
1345 * removal.
1346 *
1347 * When called with a force argument, attempt to switch all TCBs to
1348 * use the default stack instead of returning EBUSY.
1349 *
1350 * Returns 0 on success (or if the removal would succeed), or an error
1351 * code on failure.
1352 */
1353 int
deregister_tcp_functions(struct tcp_function_block * blk,bool quiesce,bool force)1354 deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce,
1355 bool force)
1356 {
1357 struct tcp_function *f;
1358 VNET_ITERATOR_DECL(vnet_iter);
1359
1360 if (blk == &tcp_def_funcblk) {
1361 /* You can't un-register the default */
1362 return (EPERM);
1363 }
1364 rw_wlock(&tcp_function_lock);
1365 VNET_LIST_RLOCK_NOSLEEP();
1366 VNET_FOREACH(vnet_iter) {
1367 CURVNET_SET(vnet_iter);
1368 if (blk == V_tcp_func_set_ptr) {
1369 /* You can't free the current default in some vnet. */
1370 CURVNET_RESTORE();
1371 VNET_LIST_RUNLOCK_NOSLEEP();
1372 rw_wunlock(&tcp_function_lock);
1373 return (EBUSY);
1374 }
1375 CURVNET_RESTORE();
1376 }
1377 VNET_LIST_RUNLOCK_NOSLEEP();
1378 /* Mark the block so no more stacks can use it. */
1379 blk->tfb_flags |= TCP_FUNC_BEING_REMOVED;
1380 /*
1381 * If TCBs are still attached to the stack, attempt to switch them
1382 * to the default stack.
1383 */
1384 if (force && blk->tfb_refcnt) {
1385 struct inpcb *inp;
1386 struct tcpcb *tp;
1387 VNET_ITERATOR_DECL(vnet_iter);
1388
1389 rw_wunlock(&tcp_function_lock);
1390
1391 VNET_LIST_RLOCK();
1392 VNET_FOREACH(vnet_iter) {
1393 CURVNET_SET(vnet_iter);
1394 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1395 INPLOOKUP_WLOCKPCB);
1396
1397 while ((inp = inp_next(&inpi)) != NULL) {
1398 tp = intotcpcb(inp);
1399 if (tp == NULL || tp->t_fb != blk)
1400 continue;
1401 tcp_switch_back_to_default(tp);
1402 }
1403 CURVNET_RESTORE();
1404 }
1405 VNET_LIST_RUNLOCK();
1406
1407 rw_wlock(&tcp_function_lock);
1408 }
1409 if (blk->tfb_refcnt) {
1410 /* TCBs still attached. */
1411 rw_wunlock(&tcp_function_lock);
1412 return (EBUSY);
1413 }
1414 if (quiesce) {
1415 /* Skip removal. */
1416 rw_wunlock(&tcp_function_lock);
1417 return (0);
1418 }
1419 /* Remove any function names that map to this function block. */
1420 while (find_tcp_fb_locked(blk, &f) != NULL) {
1421 TAILQ_REMOVE(&t_functions, f, tf_next);
1422 tcp_fb_cnt--;
1423 f->tf_fb = NULL;
1424 free(f, M_TCPFUNCTIONS);
1425 }
1426 rw_wunlock(&tcp_function_lock);
1427 return (0);
1428 }
1429
1430 static void
tcp_drain(void)1431 tcp_drain(void)
1432 {
1433 struct epoch_tracker et;
1434 VNET_ITERATOR_DECL(vnet_iter);
1435
1436 if (!do_tcpdrain)
1437 return;
1438
1439 NET_EPOCH_ENTER(et);
1440 VNET_LIST_RLOCK_NOSLEEP();
1441 VNET_FOREACH(vnet_iter) {
1442 CURVNET_SET(vnet_iter);
1443 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1444 INPLOOKUP_WLOCKPCB);
1445 struct inpcb *inpb;
1446 struct tcpcb *tcpb;
1447
1448 /*
1449 * Walk the tcpbs, if existing, and flush the reassembly queue,
1450 * if there is one...
1451 * XXX: The "Net/3" implementation doesn't imply that the TCP
1452 * reassembly queue should be flushed, but in a situation
1453 * where we're really low on mbufs, this is potentially
1454 * useful.
1455 */
1456 while ((inpb = inp_next(&inpi)) != NULL) {
1457 if ((tcpb = intotcpcb(inpb)) != NULL) {
1458 tcp_reass_flush(tcpb);
1459 tcp_clean_sackreport(tcpb);
1460 #ifdef TCP_BLACKBOX
1461 tcp_log_drain(tcpb);
1462 #endif
1463 #ifdef TCPPCAP
1464 if (tcp_pcap_aggressive_free) {
1465 /* Free the TCP PCAP queues. */
1466 tcp_pcap_drain(&(tcpb->t_inpkts));
1467 tcp_pcap_drain(&(tcpb->t_outpkts));
1468 }
1469 #endif
1470 }
1471 }
1472 CURVNET_RESTORE();
1473 }
1474 VNET_LIST_RUNLOCK_NOSLEEP();
1475 NET_EPOCH_EXIT(et);
1476 }
1477
1478 static void
tcp_vnet_init(void * arg __unused)1479 tcp_vnet_init(void *arg __unused)
1480 {
1481
1482 #ifdef TCP_HHOOK
1483 if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
1484 &V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1485 printf("%s: WARNING: unable to register helper hook\n", __func__);
1486 if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
1487 &V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1488 printf("%s: WARNING: unable to register helper hook\n", __func__);
1489 #endif
1490 #ifdef STATS
1491 if (tcp_stats_init())
1492 printf("%s: WARNING: unable to initialise TCP stats\n",
1493 __func__);
1494 #endif
1495 in_pcbinfo_init(&V_tcbinfo, &tcpcbstor, tcp_tcbhashsize,
1496 tcp_tcbhashsize);
1497
1498 syncache_init();
1499 tcp_hc_init();
1500
1501 TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
1502 V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
1503 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1504
1505 tcp_fastopen_init();
1506
1507 COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK);
1508 VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK);
1509
1510 V_tcp_msl = TCPTV_MSL;
1511 arc4rand(&V_ts_offset_secret, sizeof(V_ts_offset_secret), 0);
1512 }
1513 VNET_SYSINIT(tcp_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
1514 tcp_vnet_init, NULL);
1515
1516 static void
tcp_init(void * arg __unused)1517 tcp_init(void *arg __unused)
1518 {
1519 int hashsize;
1520
1521 tcp_reass_global_init();
1522
1523 /* XXX virtualize those below? */
1524 tcp_delacktime = TCPTV_DELACK;
1525 tcp_keepinit = TCPTV_KEEP_INIT;
1526 tcp_keepidle = TCPTV_KEEP_IDLE;
1527 tcp_keepintvl = TCPTV_KEEPINTVL;
1528 tcp_maxpersistidle = TCPTV_KEEP_IDLE;
1529 tcp_rexmit_initial = TCPTV_RTOBASE;
1530 if (tcp_rexmit_initial < 1)
1531 tcp_rexmit_initial = 1;
1532 tcp_rexmit_min = TCPTV_MIN;
1533 if (tcp_rexmit_min < 1)
1534 tcp_rexmit_min = 1;
1535 tcp_persmin = TCPTV_PERSMIN;
1536 tcp_persmax = TCPTV_PERSMAX;
1537 tcp_rexmit_slop = TCPTV_CPU_VAR;
1538 tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
1539
1540 /* Setup the tcp function block list */
1541 TAILQ_INIT(&t_functions);
1542 rw_init(&tcp_function_lock, "tcp_func_lock");
1543 register_tcp_functions(&tcp_def_funcblk, M_WAITOK);
1544 sx_init(&tcpoudp_lock, "TCP over UDP configuration");
1545 #ifdef TCP_BLACKBOX
1546 /* Initialize the TCP logging data. */
1547 tcp_log_init();
1548 #endif
1549
1550 if (tcp_soreceive_stream) {
1551 #ifdef INET
1552 tcp_protosw.pr_soreceive = soreceive_stream;
1553 #endif
1554 #ifdef INET6
1555 tcp6_protosw.pr_soreceive = soreceive_stream;
1556 #endif /* INET6 */
1557 }
1558
1559 #ifdef INET6
1560 max_protohdr_grow(sizeof(struct ip6_hdr) + sizeof(struct tcphdr));
1561 #else /* INET6 */
1562 max_protohdr_grow(sizeof(struct tcpiphdr));
1563 #endif /* INET6 */
1564
1565 ISN_LOCK_INIT();
1566 EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
1567 SHUTDOWN_PRI_DEFAULT);
1568 EVENTHANDLER_REGISTER(vm_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1569 EVENTHANDLER_REGISTER(mbuf_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1570
1571 tcp_inp_lro_direct_queue = counter_u64_alloc(M_WAITOK);
1572 tcp_inp_lro_wokeup_queue = counter_u64_alloc(M_WAITOK);
1573 tcp_inp_lro_compressed = counter_u64_alloc(M_WAITOK);
1574 tcp_inp_lro_locks_taken = counter_u64_alloc(M_WAITOK);
1575 tcp_extra_mbuf = counter_u64_alloc(M_WAITOK);
1576 tcp_would_have_but = counter_u64_alloc(M_WAITOK);
1577 tcp_comp_total = counter_u64_alloc(M_WAITOK);
1578 tcp_uncomp_total = counter_u64_alloc(M_WAITOK);
1579 tcp_bad_csums = counter_u64_alloc(M_WAITOK);
1580 tcp_pacing_failures = counter_u64_alloc(M_WAITOK);
1581 #ifdef TCPPCAP
1582 tcp_pcap_init();
1583 #endif
1584
1585 hashsize = tcp_tcbhashsize;
1586 if (hashsize == 0) {
1587 /*
1588 * Auto tune the hash size based on maxsockets.
1589 * A perfect hash would have a 1:1 mapping
1590 * (hashsize = maxsockets) however it's been
1591 * suggested that O(2) average is better.
1592 */
1593 hashsize = maketcp_hashsize(maxsockets / 4);
1594 /*
1595 * Our historical default is 512,
1596 * do not autotune lower than this.
1597 */
1598 if (hashsize < 512)
1599 hashsize = 512;
1600 if (bootverbose)
1601 printf("%s: %s auto tuned to %d\n", __func__,
1602 "net.inet.tcp.tcbhashsize", hashsize);
1603 }
1604 /*
1605 * We require a hashsize to be a power of two.
1606 * Previously if it was not a power of two we would just reset it
1607 * back to 512, which could be a nasty surprise if you did not notice
1608 * the error message.
1609 * Instead what we do is clip it to the closest power of two lower
1610 * than the specified hash value.
1611 */
1612 if (!powerof2(hashsize)) {
1613 int oldhashsize = hashsize;
1614
1615 hashsize = maketcp_hashsize(hashsize);
1616 /* prevent absurdly low value */
1617 if (hashsize < 16)
1618 hashsize = 16;
1619 printf("%s: WARNING: TCB hash size not a power of 2, "
1620 "clipped from %d to %d.\n", __func__, oldhashsize,
1621 hashsize);
1622 }
1623 tcp_tcbhashsize = hashsize;
1624
1625 #ifdef INET
1626 IPPROTO_REGISTER(IPPROTO_TCP, tcp_input, tcp_ctlinput);
1627 #endif
1628 #ifdef INET6
1629 IP6PROTO_REGISTER(IPPROTO_TCP, tcp6_input, tcp6_ctlinput);
1630 #endif
1631 }
1632 SYSINIT(tcp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, tcp_init, NULL);
1633
1634 #ifdef VIMAGE
1635 static void
tcp_destroy(void * unused __unused)1636 tcp_destroy(void *unused __unused)
1637 {
1638 int n;
1639 #ifdef TCP_HHOOK
1640 int error;
1641 #endif
1642
1643 /*
1644 * All our processes are gone, all our sockets should be cleaned
1645 * up, which means, we should be past the tcp_discardcb() calls.
1646 * Sleep to let all tcpcb timers really disappear and cleanup.
1647 */
1648 for (;;) {
1649 INP_INFO_WLOCK(&V_tcbinfo);
1650 n = V_tcbinfo.ipi_count;
1651 INP_INFO_WUNLOCK(&V_tcbinfo);
1652 if (n == 0)
1653 break;
1654 pause("tcpdes", hz / 10);
1655 }
1656 tcp_hc_destroy();
1657 syncache_destroy();
1658 in_pcbinfo_destroy(&V_tcbinfo);
1659 /* tcp_discardcb() clears the sack_holes up. */
1660 uma_zdestroy(V_sack_hole_zone);
1661
1662 /*
1663 * Cannot free the zone until all tcpcbs are released as we attach
1664 * the allocations to them.
1665 */
1666 tcp_fastopen_destroy();
1667
1668 COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES);
1669 VNET_PCPUSTAT_FREE(tcpstat);
1670
1671 #ifdef TCP_HHOOK
1672 error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
1673 if (error != 0) {
1674 printf("%s: WARNING: unable to deregister helper hook "
1675 "type=%d, id=%d: error %d returned\n", __func__,
1676 HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
1677 }
1678 error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
1679 if (error != 0) {
1680 printf("%s: WARNING: unable to deregister helper hook "
1681 "type=%d, id=%d: error %d returned\n", __func__,
1682 HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
1683 }
1684 #endif
1685 }
1686 VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
1687 #endif
1688
1689 void
tcp_fini(void * xtp)1690 tcp_fini(void *xtp)
1691 {
1692
1693 }
1694
1695 /*
1696 * Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
1697 * tcp_template used to store this data in mbufs, but we now recopy it out
1698 * of the tcpcb each time to conserve mbufs.
1699 */
1700 void
tcpip_fillheaders(struct inpcb * inp,uint16_t port,void * ip_ptr,void * tcp_ptr)1701 tcpip_fillheaders(struct inpcb *inp, uint16_t port, void *ip_ptr, void *tcp_ptr)
1702 {
1703 struct tcphdr *th = (struct tcphdr *)tcp_ptr;
1704
1705 INP_WLOCK_ASSERT(inp);
1706
1707 #ifdef INET6
1708 if ((inp->inp_vflag & INP_IPV6) != 0) {
1709 struct ip6_hdr *ip6;
1710
1711 ip6 = (struct ip6_hdr *)ip_ptr;
1712 ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
1713 (inp->inp_flow & IPV6_FLOWINFO_MASK);
1714 ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) |
1715 (IPV6_VERSION & IPV6_VERSION_MASK);
1716 if (port == 0)
1717 ip6->ip6_nxt = IPPROTO_TCP;
1718 else
1719 ip6->ip6_nxt = IPPROTO_UDP;
1720 ip6->ip6_plen = htons(sizeof(struct tcphdr));
1721 ip6->ip6_src = inp->in6p_laddr;
1722 ip6->ip6_dst = inp->in6p_faddr;
1723 }
1724 #endif /* INET6 */
1725 #if defined(INET6) && defined(INET)
1726 else
1727 #endif
1728 #ifdef INET
1729 {
1730 struct ip *ip;
1731
1732 ip = (struct ip *)ip_ptr;
1733 ip->ip_v = IPVERSION;
1734 ip->ip_hl = 5;
1735 ip->ip_tos = inp->inp_ip_tos;
1736 ip->ip_len = 0;
1737 ip->ip_id = 0;
1738 ip->ip_off = 0;
1739 ip->ip_ttl = inp->inp_ip_ttl;
1740 ip->ip_sum = 0;
1741 if (port == 0)
1742 ip->ip_p = IPPROTO_TCP;
1743 else
1744 ip->ip_p = IPPROTO_UDP;
1745 ip->ip_src = inp->inp_laddr;
1746 ip->ip_dst = inp->inp_faddr;
1747 }
1748 #endif /* INET */
1749 th->th_sport = inp->inp_lport;
1750 th->th_dport = inp->inp_fport;
1751 th->th_seq = 0;
1752 th->th_ack = 0;
1753 th->th_off = 5;
1754 tcp_set_flags(th, 0);
1755 th->th_win = 0;
1756 th->th_urp = 0;
1757 th->th_sum = 0; /* in_pseudo() is called later for ipv4 */
1758 }
1759
1760 /*
1761 * Create template to be used to send tcp packets on a connection.
1762 * Allocates an mbuf and fills in a skeletal tcp/ip header. The only
1763 * use for this function is in keepalives, which use tcp_respond.
1764 */
1765 struct tcptemp *
tcpip_maketemplate(struct inpcb * inp)1766 tcpip_maketemplate(struct inpcb *inp)
1767 {
1768 struct tcptemp *t;
1769
1770 t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
1771 if (t == NULL)
1772 return (NULL);
1773 tcpip_fillheaders(inp, 0, (void *)&t->tt_ipgen, (void *)&t->tt_t);
1774 return (t);
1775 }
1776
1777 /*
1778 * Send a single message to the TCP at address specified by
1779 * the given TCP/IP header. If m == NULL, then we make a copy
1780 * of the tcpiphdr at th and send directly to the addressed host.
1781 * This is used to force keep alive messages out using the TCP
1782 * template for a connection. If flags are given then we send
1783 * a message back to the TCP which originated the segment th,
1784 * and discard the mbuf containing it and any other attached mbufs.
1785 *
1786 * In any case the ack and sequence number of the transmitted
1787 * segment are as specified by the parameters.
1788 *
1789 * NOTE: If m != NULL, then th must point to *inside* the mbuf.
1790 */
1791 void
tcp_respond(struct tcpcb * tp,void * ipgen,struct tcphdr * th,struct mbuf * m,tcp_seq ack,tcp_seq seq,uint16_t flags)1792 tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
1793 tcp_seq ack, tcp_seq seq, uint16_t flags)
1794 {
1795 struct tcpopt to;
1796 struct inpcb *inp;
1797 struct ip *ip;
1798 struct mbuf *optm;
1799 struct udphdr *uh = NULL;
1800 struct tcphdr *nth;
1801 struct tcp_log_buffer *lgb;
1802 u_char *optp;
1803 #ifdef INET6
1804 struct ip6_hdr *ip6;
1805 int isipv6;
1806 #endif /* INET6 */
1807 int optlen, tlen, win, ulen;
1808 int ect = 0;
1809 bool incl_opts;
1810 uint16_t port;
1811 int output_ret;
1812 #ifdef INVARIANTS
1813 int thflags = tcp_get_flags(th);
1814 #endif
1815
1816 KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL"));
1817 NET_EPOCH_ASSERT();
1818
1819 #ifdef INET6
1820 isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
1821 ip6 = ipgen;
1822 #endif /* INET6 */
1823 ip = ipgen;
1824
1825 if (tp != NULL) {
1826 inp = tptoinpcb(tp);
1827 INP_LOCK_ASSERT(inp);
1828 } else
1829 inp = NULL;
1830
1831 if (m != NULL) {
1832 #ifdef INET6
1833 if (isipv6 && ip6 && (ip6->ip6_nxt == IPPROTO_UDP))
1834 port = m->m_pkthdr.tcp_tun_port;
1835 else
1836 #endif
1837 if (ip && (ip->ip_p == IPPROTO_UDP))
1838 port = m->m_pkthdr.tcp_tun_port;
1839 else
1840 port = 0;
1841 } else
1842 port = tp->t_port;
1843
1844 incl_opts = false;
1845 win = 0;
1846 if (tp != NULL) {
1847 if (!(flags & TH_RST)) {
1848 win = sbspace(&inp->inp_socket->so_rcv);
1849 if (win > TCP_MAXWIN << tp->rcv_scale)
1850 win = TCP_MAXWIN << tp->rcv_scale;
1851 }
1852 if ((tp->t_flags & TF_NOOPT) == 0)
1853 incl_opts = true;
1854 }
1855 if (m == NULL) {
1856 m = m_gethdr(M_NOWAIT, MT_DATA);
1857 if (m == NULL)
1858 return;
1859 m->m_data += max_linkhdr;
1860 #ifdef INET6
1861 if (isipv6) {
1862 bcopy((caddr_t)ip6, mtod(m, caddr_t),
1863 sizeof(struct ip6_hdr));
1864 ip6 = mtod(m, struct ip6_hdr *);
1865 nth = (struct tcphdr *)(ip6 + 1);
1866 if (port) {
1867 /* Insert a UDP header */
1868 uh = (struct udphdr *)nth;
1869 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1870 uh->uh_dport = port;
1871 nth = (struct tcphdr *)(uh + 1);
1872 }
1873 } else
1874 #endif /* INET6 */
1875 {
1876 bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
1877 ip = mtod(m, struct ip *);
1878 nth = (struct tcphdr *)(ip + 1);
1879 if (port) {
1880 /* Insert a UDP header */
1881 uh = (struct udphdr *)nth;
1882 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1883 uh->uh_dport = port;
1884 nth = (struct tcphdr *)(uh + 1);
1885 }
1886 }
1887 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1888 flags = TH_ACK;
1889 } else if ((!M_WRITABLE(m)) || (port != 0)) {
1890 struct mbuf *n;
1891
1892 /* Can't reuse 'm', allocate a new mbuf. */
1893 n = m_gethdr(M_NOWAIT, MT_DATA);
1894 if (n == NULL) {
1895 m_freem(m);
1896 return;
1897 }
1898
1899 if (!m_dup_pkthdr(n, m, M_NOWAIT)) {
1900 m_freem(m);
1901 m_freem(n);
1902 return;
1903 }
1904
1905 n->m_data += max_linkhdr;
1906 /* m_len is set later */
1907 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
1908 #ifdef INET6
1909 if (isipv6) {
1910 bcopy((caddr_t)ip6, mtod(n, caddr_t),
1911 sizeof(struct ip6_hdr));
1912 ip6 = mtod(n, struct ip6_hdr *);
1913 xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1914 nth = (struct tcphdr *)(ip6 + 1);
1915 if (port) {
1916 /* Insert a UDP header */
1917 uh = (struct udphdr *)nth;
1918 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1919 uh->uh_dport = port;
1920 nth = (struct tcphdr *)(uh + 1);
1921 }
1922 } else
1923 #endif /* INET6 */
1924 {
1925 bcopy((caddr_t)ip, mtod(n, caddr_t), sizeof(struct ip));
1926 ip = mtod(n, struct ip *);
1927 xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1928 nth = (struct tcphdr *)(ip + 1);
1929 if (port) {
1930 /* Insert a UDP header */
1931 uh = (struct udphdr *)nth;
1932 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1933 uh->uh_dport = port;
1934 nth = (struct tcphdr *)(uh + 1);
1935 }
1936 }
1937 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1938 xchg(nth->th_dport, nth->th_sport, uint16_t);
1939 th = nth;
1940 m_freem(m);
1941 m = n;
1942 } else {
1943 /*
1944 * reuse the mbuf.
1945 * XXX MRT We inherit the FIB, which is lucky.
1946 */
1947 m_freem(m->m_next);
1948 m->m_next = NULL;
1949 m->m_data = (caddr_t)ipgen;
1950 /* clear any receive flags for proper bpf timestamping */
1951 m->m_flags &= ~(M_TSTMP | M_TSTMP_LRO);
1952 /* m_len is set later */
1953 #ifdef INET6
1954 if (isipv6) {
1955 xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1956 nth = (struct tcphdr *)(ip6 + 1);
1957 } else
1958 #endif /* INET6 */
1959 {
1960 xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1961 nth = (struct tcphdr *)(ip + 1);
1962 }
1963 if (th != nth) {
1964 /*
1965 * this is usually a case when an extension header
1966 * exists between the IPv6 header and the
1967 * TCP header.
1968 */
1969 nth->th_sport = th->th_sport;
1970 nth->th_dport = th->th_dport;
1971 }
1972 xchg(nth->th_dport, nth->th_sport, uint16_t);
1973 #undef xchg
1974 }
1975 tlen = 0;
1976 #ifdef INET6
1977 if (isipv6)
1978 tlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1979 #endif
1980 #if defined(INET) && defined(INET6)
1981 else
1982 #endif
1983 #ifdef INET
1984 tlen = sizeof (struct tcpiphdr);
1985 #endif
1986 if (port)
1987 tlen += sizeof (struct udphdr);
1988 #ifdef INVARIANTS
1989 m->m_len = 0;
1990 KASSERT(M_TRAILINGSPACE(m) >= tlen,
1991 ("Not enough trailing space for message (m=%p, need=%d, have=%ld)",
1992 m, tlen, (long)M_TRAILINGSPACE(m)));
1993 #endif
1994 m->m_len = tlen;
1995 to.to_flags = 0;
1996 if (incl_opts) {
1997 ect = tcp_ecn_output_established(tp, &flags, 0, false);
1998 /* Make sure we have room. */
1999 if (M_TRAILINGSPACE(m) < TCP_MAXOLEN) {
2000 m->m_next = m_get(M_NOWAIT, MT_DATA);
2001 if (m->m_next) {
2002 optp = mtod(m->m_next, u_char *);
2003 optm = m->m_next;
2004 } else
2005 incl_opts = false;
2006 } else {
2007 optp = (u_char *) (nth + 1);
2008 optm = m;
2009 }
2010 }
2011 if (incl_opts) {
2012 /* Timestamps. */
2013 if (tp->t_flags & TF_RCVD_TSTMP) {
2014 to.to_tsval = tcp_ts_getticks() + tp->ts_offset;
2015 to.to_tsecr = tp->ts_recent;
2016 to.to_flags |= TOF_TS;
2017 }
2018 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2019 /* TCP-MD5 (RFC2385). */
2020 if (tp->t_flags & TF_SIGNATURE)
2021 to.to_flags |= TOF_SIGNATURE;
2022 #endif
2023 /* Add the options. */
2024 tlen += optlen = tcp_addoptions(&to, optp);
2025
2026 /* Update m_len in the correct mbuf. */
2027 optm->m_len += optlen;
2028 } else
2029 optlen = 0;
2030 #ifdef INET6
2031 if (isipv6) {
2032 if (uh) {
2033 ulen = tlen - sizeof(struct ip6_hdr);
2034 uh->uh_ulen = htons(ulen);
2035 }
2036 ip6->ip6_flow = htonl(ect << IPV6_FLOWLABEL_LEN);
2037 ip6->ip6_vfc = IPV6_VERSION;
2038 if (port)
2039 ip6->ip6_nxt = IPPROTO_UDP;
2040 else
2041 ip6->ip6_nxt = IPPROTO_TCP;
2042 ip6->ip6_plen = htons(tlen - sizeof(*ip6));
2043 }
2044 #endif
2045 #if defined(INET) && defined(INET6)
2046 else
2047 #endif
2048 #ifdef INET
2049 {
2050 if (uh) {
2051 ulen = tlen - sizeof(struct ip);
2052 uh->uh_ulen = htons(ulen);
2053 }
2054 ip->ip_len = htons(tlen);
2055 if (inp != NULL) {
2056 ip->ip_tos = inp->inp_ip_tos & ~IPTOS_ECN_MASK;
2057 ip->ip_ttl = inp->inp_ip_ttl;
2058 } else {
2059 ip->ip_tos = 0;
2060 ip->ip_ttl = V_ip_defttl;
2061 }
2062 ip->ip_tos |= ect;
2063 if (port) {
2064 ip->ip_p = IPPROTO_UDP;
2065 } else {
2066 ip->ip_p = IPPROTO_TCP;
2067 }
2068 if (V_path_mtu_discovery)
2069 ip->ip_off |= htons(IP_DF);
2070 }
2071 #endif
2072 m->m_pkthdr.len = tlen;
2073 m->m_pkthdr.rcvif = NULL;
2074 #ifdef MAC
2075 if (inp != NULL) {
2076 /*
2077 * Packet is associated with a socket, so allow the
2078 * label of the response to reflect the socket label.
2079 */
2080 INP_LOCK_ASSERT(inp);
2081 mac_inpcb_create_mbuf(inp, m);
2082 } else {
2083 /*
2084 * Packet is not associated with a socket, so possibly
2085 * update the label in place.
2086 */
2087 mac_netinet_tcp_reply(m);
2088 }
2089 #endif
2090 nth->th_seq = htonl(seq);
2091 nth->th_ack = htonl(ack);
2092 nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
2093 tcp_set_flags(nth, flags);
2094 if (tp && (flags & TH_RST)) {
2095 /* Log the reset */
2096 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
2097 }
2098 if (tp != NULL)
2099 nth->th_win = htons((u_short) (win >> tp->rcv_scale));
2100 else
2101 nth->th_win = htons((u_short)win);
2102 nth->th_urp = 0;
2103
2104 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2105 if (to.to_flags & TOF_SIGNATURE) {
2106 if (!TCPMD5_ENABLED() ||
2107 TCPMD5_OUTPUT(m, nth, to.to_signature) != 0) {
2108 m_freem(m);
2109 return;
2110 }
2111 }
2112 #endif
2113
2114 #ifdef INET6
2115 if (isipv6) {
2116 if (port) {
2117 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
2118 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2119 uh->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
2120 nth->th_sum = 0;
2121 } else {
2122 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2123 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2124 nth->th_sum = in6_cksum_pseudo(ip6,
2125 tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
2126 }
2127 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
2128 }
2129 #endif /* INET6 */
2130 #if defined(INET6) && defined(INET)
2131 else
2132 #endif
2133 #ifdef INET
2134 {
2135 if (port) {
2136 uh->uh_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2137 htons(ulen + IPPROTO_UDP));
2138 m->m_pkthdr.csum_flags = CSUM_UDP;
2139 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2140 nth->th_sum = 0;
2141 } else {
2142 m->m_pkthdr.csum_flags = CSUM_TCP;
2143 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2144 nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2145 htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
2146 }
2147 }
2148 #endif /* INET */
2149 TCP_PROBE3(debug__output, tp, th, m);
2150 if (flags & TH_RST)
2151 TCP_PROBE5(accept__refused, NULL, NULL, m, tp, nth);
2152 lgb = NULL;
2153 if ((tp != NULL) && tcp_bblogging_on(tp)) {
2154 if (INP_WLOCKED(inp)) {
2155 union tcp_log_stackspecific log;
2156 struct timeval tv;
2157
2158 memset(&log, 0, sizeof(log));
2159 log.u_bbr.inhpts = tcp_in_hpts(tp);
2160 log.u_bbr.flex8 = 4;
2161 log.u_bbr.pkts_out = tp->t_maxseg;
2162 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2163 log.u_bbr.delivered = 0;
2164 lgb = tcp_log_event(tp, nth, NULL, NULL, TCP_LOG_OUT,
2165 ERRNO_UNK, 0, &log, false, NULL, NULL, 0, &tv);
2166 } else {
2167 /*
2168 * We can not log the packet, since we only own the
2169 * read lock, but a write lock is needed. The read lock
2170 * is not upgraded to a write lock, since only getting
2171 * the read lock was done intentionally to improve the
2172 * handling of SYN flooding attacks.
2173 * This happens only for pure SYN segments received in
2174 * the initial CLOSED state, or received in a more
2175 * advanced state than listen and the UDP encapsulation
2176 * port is unexpected.
2177 * The incoming SYN segments do not really belong to
2178 * the TCP connection and the handling does not change
2179 * the state of the TCP connection. Therefore, the
2180 * sending of the RST segments is not logged. Please
2181 * note that also the incoming SYN segments are not
2182 * logged.
2183 *
2184 * The following code ensures that the above description
2185 * is and stays correct.
2186 */
2187 KASSERT((thflags & (TH_ACK|TH_SYN)) == TH_SYN &&
2188 (tp->t_state == TCPS_CLOSED ||
2189 (tp->t_state > TCPS_LISTEN && tp->t_port != port)),
2190 ("%s: Logging of TCP segment with flags 0x%b and "
2191 "UDP encapsulation port %u skipped in state %s",
2192 __func__, thflags, PRINT_TH_FLAGS,
2193 ntohs(port), tcpstates[tp->t_state]));
2194 }
2195 }
2196
2197 if (flags & TH_ACK)
2198 TCPSTAT_INC(tcps_sndacks);
2199 else if (flags & (TH_SYN|TH_FIN|TH_RST))
2200 TCPSTAT_INC(tcps_sndctrl);
2201 TCPSTAT_INC(tcps_sndtotal);
2202
2203 #ifdef INET6
2204 if (isipv6) {
2205 TCP_PROBE5(send, NULL, tp, ip6, tp, nth);
2206 output_ret = ip6_output(m, inp ? inp->in6p_outputopts : NULL,
2207 NULL, 0, NULL, NULL, inp);
2208 }
2209 #endif /* INET6 */
2210 #if defined(INET) && defined(INET6)
2211 else
2212 #endif
2213 #ifdef INET
2214 {
2215 TCP_PROBE5(send, NULL, tp, ip, tp, nth);
2216 output_ret = ip_output(m, NULL, NULL, 0, NULL, inp);
2217 }
2218 #endif
2219 if (lgb != NULL)
2220 lgb->tlb_errno = output_ret;
2221 }
2222
2223 /*
2224 * Send a challenge ack (no data, no SACK option), but not more than
2225 * V_tcp_ack_war_cnt per V_tcp_ack_war_time_window (per TCP connection).
2226 */
2227 void
tcp_send_challenge_ack(struct tcpcb * tp,struct tcphdr * th,struct mbuf * m)2228 tcp_send_challenge_ack(struct tcpcb *tp, struct tcphdr *th, struct mbuf *m)
2229 {
2230 sbintime_t now;
2231 bool send_challenge_ack;
2232
2233 if (V_tcp_ack_war_time_window == 0 || V_tcp_ack_war_cnt == 0) {
2234 /* ACK war protection is disabled. */
2235 send_challenge_ack = true;
2236 } else {
2237 /* Start new epoch, if the previous one is already over. */
2238 now = getsbinuptime();
2239 if (tp->t_challenge_ack_end < now) {
2240 tp->t_challenge_ack_cnt = 0;
2241 tp->t_challenge_ack_end = now +
2242 V_tcp_ack_war_time_window * SBT_1MS;
2243 }
2244 /*
2245 * Send a challenge ACK, if less than tcp_ack_war_cnt have been
2246 * sent in the current epoch.
2247 */
2248 if (tp->t_challenge_ack_cnt < V_tcp_ack_war_cnt) {
2249 send_challenge_ack = true;
2250 tp->t_challenge_ack_cnt++;
2251 } else {
2252 send_challenge_ack = false;
2253 }
2254 }
2255 if (send_challenge_ack) {
2256 tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt,
2257 tp->snd_nxt, TH_ACK);
2258 tp->last_ack_sent = tp->rcv_nxt;
2259 }
2260 }
2261
2262 /*
2263 * Create a new TCP control block, making an empty reassembly queue and hooking
2264 * it to the argument protocol control block. The `inp' parameter must have
2265 * come from the zone allocator set up by tcpcbstor declaration.
2266 * The caller can provide a pointer to a tcpcb of the listener to inherit the
2267 * TCP function block from the listener.
2268 */
2269 struct tcpcb *
tcp_newtcpcb(struct inpcb * inp,struct tcpcb * listening_tcb)2270 tcp_newtcpcb(struct inpcb *inp, struct tcpcb *listening_tcb)
2271 {
2272 struct tcpcb *tp = intotcpcb(inp);
2273 #ifdef INET6
2274 int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2275 #endif /* INET6 */
2276
2277 /*
2278 * Historically allocation was done with M_ZERO. There is a lot of
2279 * code that rely on that. For now take safe approach and zero whole
2280 * tcpcb. This definitely can be optimized.
2281 */
2282 bzero(&tp->t_start_zero, t_zero_size);
2283
2284 /* Initialise cc_var struct for this tcpcb. */
2285 tp->t_ccv.type = IPPROTO_TCP;
2286 tp->t_ccv.ccvc.tcp = tp;
2287 rw_rlock(&tcp_function_lock);
2288 if (listening_tcb != NULL) {
2289 INP_LOCK_ASSERT(tptoinpcb(listening_tcb));
2290 KASSERT(listening_tcb->t_fb != NULL,
2291 ("tcp_newtcpcb: listening_tcb->t_fb is NULL"));
2292 if (listening_tcb->t_fb->tfb_flags & TCP_FUNC_BEING_REMOVED) {
2293 rw_runlock(&tcp_function_lock);
2294 return (NULL);
2295 }
2296 tp->t_fb = listening_tcb->t_fb;
2297 } else {
2298 tp->t_fb = V_tcp_func_set_ptr;
2299 }
2300 refcount_acquire(&tp->t_fb->tfb_refcnt);
2301 KASSERT((tp->t_fb->tfb_flags & TCP_FUNC_BEING_REMOVED) == 0,
2302 ("tcp_newtcpcb: using TFB being removed"));
2303 rw_runlock(&tcp_function_lock);
2304 /*
2305 * Use the current system default CC algorithm.
2306 */
2307 cc_attach(tp, CC_DEFAULT_ALGO());
2308
2309 if (CC_ALGO(tp)->cb_init != NULL)
2310 if (CC_ALGO(tp)->cb_init(&tp->t_ccv, NULL) > 0) {
2311 cc_detach(tp);
2312 if (tp->t_fb->tfb_tcp_fb_fini)
2313 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2314 refcount_release(&tp->t_fb->tfb_refcnt);
2315 return (NULL);
2316 }
2317
2318 #ifdef TCP_HHOOK
2319 if (khelp_init_osd(HELPER_CLASS_TCP, &tp->t_osd)) {
2320 if (CC_ALGO(tp)->cb_destroy != NULL)
2321 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2322 CC_DATA(tp) = NULL;
2323 cc_detach(tp);
2324 if (tp->t_fb->tfb_tcp_fb_fini)
2325 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2326 refcount_release(&tp->t_fb->tfb_refcnt);
2327 return (NULL);
2328 }
2329 #endif
2330
2331 TAILQ_INIT(&tp->t_segq);
2332 STAILQ_INIT(&tp->t_inqueue);
2333 tp->t_maxseg =
2334 #ifdef INET6
2335 isipv6 ? V_tcp_v6mssdflt :
2336 #endif /* INET6 */
2337 V_tcp_mssdflt;
2338
2339 /* All mbuf queue/ack compress flags should be off */
2340 tcp_lro_features_off(tp);
2341
2342 tp->t_hpts_cpu = HPTS_CPU_NONE;
2343 tp->t_lro_cpu = HPTS_CPU_NONE;
2344
2345 callout_init_rw(&tp->t_callout, &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
2346 for (int i = 0; i < TT_N; i++)
2347 tp->t_timers[i] = SBT_MAX;
2348
2349 switch (V_tcp_do_rfc1323) {
2350 case 0:
2351 break;
2352 default:
2353 case 1:
2354 tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
2355 break;
2356 case 2:
2357 tp->t_flags = TF_REQ_SCALE;
2358 break;
2359 case 3:
2360 tp->t_flags = TF_REQ_TSTMP;
2361 break;
2362 }
2363 if (V_tcp_do_sack)
2364 tp->t_flags |= TF_SACK_PERMIT;
2365 TAILQ_INIT(&tp->snd_holes);
2366
2367 /*
2368 * Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
2369 * rtt estimate. Set rttvar so that srtt + 4 * rttvar gives
2370 * reasonable initial retransmit time.
2371 */
2372 tp->t_srtt = TCPTV_SRTTBASE;
2373 tp->t_rttvar = ((tcp_rexmit_initial - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
2374 tp->t_rttmin = tcp_rexmit_min;
2375 tp->t_rxtcur = tcp_rexmit_initial;
2376 tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2377 tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2378 tp->t_rcvtime = ticks;
2379 /* We always start with ticks granularity */
2380 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
2381 /*
2382 * IPv4 TTL initialization is necessary for an IPv6 socket as well,
2383 * because the socket may be bound to an IPv6 wildcard address,
2384 * which may match an IPv4-mapped IPv6 address.
2385 */
2386 inp->inp_ip_ttl = V_ip_defttl;
2387 #ifdef TCPPCAP
2388 /*
2389 * Init the TCP PCAP queues.
2390 */
2391 tcp_pcap_tcpcb_init(tp);
2392 #endif
2393 #ifdef TCP_BLACKBOX
2394 /* Initialize the per-TCPCB log data. */
2395 tcp_log_tcpcbinit(tp);
2396 #endif
2397 tp->t_pacing_rate = -1;
2398 if (tp->t_fb->tfb_tcp_fb_init) {
2399 if ((*tp->t_fb->tfb_tcp_fb_init)(tp, &tp->t_fb_ptr)) {
2400 if (CC_ALGO(tp)->cb_destroy != NULL)
2401 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2402 CC_DATA(tp) = NULL;
2403 cc_detach(tp);
2404 #ifdef TCP_HHOOK
2405 khelp_destroy_osd(&tp->t_osd);
2406 #endif
2407 refcount_release(&tp->t_fb->tfb_refcnt);
2408 return (NULL);
2409 }
2410 }
2411 #ifdef STATS
2412 if (V_tcp_perconn_stats_enable == 1)
2413 tp->t_stats = stats_blob_alloc(V_tcp_perconn_stats_dflt_tpl, 0);
2414 #endif
2415 if (V_tcp_do_lrd)
2416 tp->t_flags |= TF_LRD;
2417
2418 return (tp);
2419 }
2420
2421 /*
2422 * Drop a TCP connection, reporting
2423 * the specified error. If connection is synchronized,
2424 * then send a RST to peer.
2425 */
2426 struct tcpcb *
tcp_drop(struct tcpcb * tp,int errno)2427 tcp_drop(struct tcpcb *tp, int errno)
2428 {
2429 struct socket *so = tptosocket(tp);
2430
2431 NET_EPOCH_ASSERT();
2432 INP_WLOCK_ASSERT(tptoinpcb(tp));
2433
2434 if (TCPS_HAVERCVDSYN(tp->t_state)) {
2435 tcp_state_change(tp, TCPS_CLOSED);
2436 /* Don't use tcp_output() here due to possible recursion. */
2437 (void)tcp_output_nodrop(tp);
2438 TCPSTAT_INC(tcps_drops);
2439 } else
2440 TCPSTAT_INC(tcps_conndrops);
2441 if (errno == ETIMEDOUT && tp->t_softerror)
2442 errno = tp->t_softerror;
2443 so->so_error = errno;
2444 return (tcp_close(tp));
2445 }
2446
2447 void
tcp_discardcb(struct tcpcb * tp)2448 tcp_discardcb(struct tcpcb *tp)
2449 {
2450 struct inpcb *inp = tptoinpcb(tp);
2451 struct socket *so = tptosocket(tp);
2452 struct mbuf *m;
2453 #ifdef INET6
2454 bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2455 #endif
2456
2457 INP_WLOCK_ASSERT(inp);
2458
2459 tcp_timer_stop(tp);
2460
2461 /* free the reassembly queue, if any */
2462 tcp_reass_flush(tp);
2463
2464 #ifdef TCP_OFFLOAD
2465 /* Disconnect offload device, if any. */
2466 if (tp->t_flags & TF_TOE)
2467 tcp_offload_detach(tp);
2468 #endif
2469
2470 tcp_free_sackholes(tp);
2471
2472 #ifdef TCPPCAP
2473 /* Free the TCP PCAP queues. */
2474 tcp_pcap_drain(&(tp->t_inpkts));
2475 tcp_pcap_drain(&(tp->t_outpkts));
2476 #endif
2477
2478 /* Allow the CC algorithm to clean up after itself. */
2479 if (CC_ALGO(tp)->cb_destroy != NULL)
2480 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2481 CC_DATA(tp) = NULL;
2482 /* Detach from the CC algorithm */
2483 cc_detach(tp);
2484
2485 #ifdef TCP_HHOOK
2486 khelp_destroy_osd(&tp->t_osd);
2487 #endif
2488 #ifdef STATS
2489 stats_blob_destroy(tp->t_stats);
2490 #endif
2491
2492 CC_ALGO(tp) = NULL;
2493 if ((m = STAILQ_FIRST(&tp->t_inqueue)) != NULL) {
2494 struct mbuf *prev;
2495
2496 STAILQ_INIT(&tp->t_inqueue);
2497 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, prev)
2498 m_freem(m);
2499 }
2500 TCPSTATES_DEC(tp->t_state);
2501
2502 if (tp->t_fb->tfb_tcp_fb_fini)
2503 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2504 MPASS(!tcp_in_hpts(tp));
2505 #ifdef TCP_BLACKBOX
2506 tcp_log_tcpcbfini(tp);
2507 #endif
2508
2509 /*
2510 * If we got enough samples through the srtt filter,
2511 * save the rtt and rttvar in the routing entry.
2512 * 'Enough' is arbitrarily defined as 4 rtt samples.
2513 * 4 samples is enough for the srtt filter to converge
2514 * to within enough % of the correct value; fewer samples
2515 * and we could save a bogus rtt. The danger is not high
2516 * as tcp quickly recovers from everything.
2517 * XXX: Works very well but needs some more statistics!
2518 *
2519 * XXXRRS: Updating must be after the stack fini() since
2520 * that may be converting some internal representation of
2521 * say srtt etc into the general one used by other stacks.
2522 * Lets also at least protect against the so being NULL
2523 * as RW stated below.
2524 */
2525 if ((tp->t_rttupdated >= 4) && (so != NULL)) {
2526 struct hc_metrics_lite metrics;
2527 uint32_t ssthresh;
2528
2529 bzero(&metrics, sizeof(metrics));
2530 /*
2531 * Update the ssthresh always when the conditions below
2532 * are satisfied. This gives us better new start value
2533 * for the congestion avoidance for new connections.
2534 * ssthresh is only set if packet loss occurred on a session.
2535 *
2536 * XXXRW: 'so' may be NULL here, and/or socket buffer may be
2537 * being torn down. Ideally this code would not use 'so'.
2538 */
2539 ssthresh = tp->snd_ssthresh;
2540 if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
2541 /*
2542 * convert the limit from user data bytes to
2543 * packets then to packet data bytes.
2544 */
2545 ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
2546 if (ssthresh < 2)
2547 ssthresh = 2;
2548 ssthresh *= (tp->t_maxseg +
2549 #ifdef INET6
2550 (isipv6 ? sizeof (struct ip6_hdr) +
2551 sizeof (struct tcphdr) :
2552 #endif
2553 sizeof (struct tcpiphdr)
2554 #ifdef INET6
2555 )
2556 #endif
2557 );
2558 } else
2559 ssthresh = 0;
2560 metrics.rmx_ssthresh = ssthresh;
2561
2562 metrics.rmx_rtt = tp->t_srtt;
2563 metrics.rmx_rttvar = tp->t_rttvar;
2564 metrics.rmx_cwnd = tp->snd_cwnd;
2565 metrics.rmx_sendpipe = 0;
2566 metrics.rmx_recvpipe = 0;
2567
2568 tcp_hc_update(&inp->inp_inc, &metrics);
2569 }
2570
2571 refcount_release(&tp->t_fb->tfb_refcnt);
2572 }
2573
2574 /*
2575 * Attempt to close a TCP control block, marking it as dropped, and freeing
2576 * the socket if we hold the only reference.
2577 */
2578 struct tcpcb *
tcp_close(struct tcpcb * tp)2579 tcp_close(struct tcpcb *tp)
2580 {
2581 struct inpcb *inp = tptoinpcb(tp);
2582 struct socket *so = tptosocket(tp);
2583
2584 INP_WLOCK_ASSERT(inp);
2585
2586 #ifdef TCP_OFFLOAD
2587 if (tp->t_state == TCPS_LISTEN)
2588 tcp_offload_listen_stop(tp);
2589 #endif
2590 /*
2591 * This releases the TFO pending counter resource for TFO listen
2592 * sockets as well as passively-created TFO sockets that transition
2593 * from SYN_RECEIVED to CLOSED.
2594 */
2595 if (tp->t_tfo_pending) {
2596 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
2597 tp->t_tfo_pending = NULL;
2598 }
2599 if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
2600 tp->t_fb->tfb_tcp_timer_stop_all(tp);
2601 in_pcbdrop(inp);
2602 TCPSTAT_INC(tcps_closed);
2603 if (tp->t_state != TCPS_CLOSED)
2604 tcp_state_change(tp, TCPS_CLOSED);
2605 KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
2606 soisdisconnected(so);
2607 if (inp->inp_flags & INP_SOCKREF) {
2608 inp->inp_flags &= ~INP_SOCKREF;
2609 INP_WUNLOCK(inp);
2610 sorele(so);
2611 return (NULL);
2612 }
2613 return (tp);
2614 }
2615
2616 /*
2617 * Notify a tcp user of an asynchronous error;
2618 * store error as soft error, but wake up user
2619 * (for now, won't do anything until can select for soft error).
2620 *
2621 * Do not wake up user since there currently is no mechanism for
2622 * reporting soft errors (yet - a kqueue filter may be added).
2623 */
2624 static struct inpcb *
tcp_notify(struct inpcb * inp,int error)2625 tcp_notify(struct inpcb *inp, int error)
2626 {
2627 struct tcpcb *tp;
2628
2629 INP_WLOCK_ASSERT(inp);
2630
2631 tp = intotcpcb(inp);
2632 KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
2633
2634 /*
2635 * Ignore some errors if we are hooked up.
2636 * If connection hasn't completed, has retransmitted several times,
2637 * and receives a second error, give up now. This is better
2638 * than waiting a long time to establish a connection that
2639 * can never complete.
2640 */
2641 if (tp->t_state == TCPS_ESTABLISHED &&
2642 (error == EHOSTUNREACH || error == ENETUNREACH ||
2643 error == EHOSTDOWN)) {
2644 if (inp->inp_route.ro_nh) {
2645 NH_FREE(inp->inp_route.ro_nh);
2646 inp->inp_route.ro_nh = (struct nhop_object *)NULL;
2647 }
2648 return (inp);
2649 } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
2650 tp->t_softerror) {
2651 tp = tcp_drop(tp, error);
2652 if (tp != NULL)
2653 return (inp);
2654 else
2655 return (NULL);
2656 } else {
2657 tp->t_softerror = error;
2658 return (inp);
2659 }
2660 #if 0
2661 wakeup( &so->so_timeo);
2662 sorwakeup(so);
2663 sowwakeup(so);
2664 #endif
2665 }
2666
2667 static int
tcp_pcblist(SYSCTL_HANDLER_ARGS)2668 tcp_pcblist(SYSCTL_HANDLER_ARGS)
2669 {
2670 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
2671 INPLOOKUP_RLOCKPCB);
2672 struct xinpgen xig;
2673 struct inpcb *inp;
2674 int error;
2675
2676 if (req->newptr != NULL)
2677 return (EPERM);
2678
2679 if (req->oldptr == NULL) {
2680 int n;
2681
2682 n = V_tcbinfo.ipi_count +
2683 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2684 n += imax(n / 8, 10);
2685 req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
2686 return (0);
2687 }
2688
2689 if ((error = sysctl_wire_old_buffer(req, 0)) != 0)
2690 return (error);
2691
2692 bzero(&xig, sizeof(xig));
2693 xig.xig_len = sizeof xig;
2694 xig.xig_count = V_tcbinfo.ipi_count +
2695 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2696 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2697 xig.xig_sogen = so_gencnt;
2698 error = SYSCTL_OUT(req, &xig, sizeof xig);
2699 if (error)
2700 return (error);
2701
2702 error = syncache_pcblist(req);
2703 if (error)
2704 return (error);
2705
2706 while ((inp = inp_next(&inpi)) != NULL) {
2707 if (inp->inp_gencnt <= xig.xig_gen &&
2708 cr_canseeinpcb(req->td->td_ucred, inp) == 0) {
2709 struct xtcpcb xt;
2710
2711 tcp_inptoxtp(inp, &xt);
2712 error = SYSCTL_OUT(req, &xt, sizeof xt);
2713 if (error) {
2714 INP_RUNLOCK(inp);
2715 break;
2716 } else
2717 continue;
2718 }
2719 }
2720
2721 if (!error) {
2722 /*
2723 * Give the user an updated idea of our state.
2724 * If the generation differs from what we told
2725 * her before, she knows that something happened
2726 * while we were processing this request, and it
2727 * might be necessary to retry.
2728 */
2729 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2730 xig.xig_sogen = so_gencnt;
2731 xig.xig_count = V_tcbinfo.ipi_count +
2732 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2733 error = SYSCTL_OUT(req, &xig, sizeof xig);
2734 }
2735
2736 return (error);
2737 }
2738
2739 SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
2740 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2741 NULL, 0, tcp_pcblist, "S,xtcpcb",
2742 "List of active TCP connections");
2743
2744 #ifdef INET
2745 static int
tcp_getcred(SYSCTL_HANDLER_ARGS)2746 tcp_getcred(SYSCTL_HANDLER_ARGS)
2747 {
2748 struct xucred xuc;
2749 struct sockaddr_in addrs[2];
2750 struct epoch_tracker et;
2751 struct inpcb *inp;
2752 int error;
2753
2754 if (req->newptr == NULL)
2755 return (EINVAL);
2756 error = priv_check(req->td, PRIV_NETINET_GETCRED);
2757 if (error)
2758 return (error);
2759 error = SYSCTL_IN(req, addrs, sizeof(addrs));
2760 if (error)
2761 return (error);
2762 NET_EPOCH_ENTER(et);
2763 inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
2764 addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
2765 NET_EPOCH_EXIT(et);
2766 if (inp != NULL) {
2767 if (error == 0)
2768 error = cr_canseeinpcb(req->td->td_ucred, inp);
2769 if (error == 0)
2770 cru2x(inp->inp_cred, &xuc);
2771 INP_RUNLOCK(inp);
2772 } else
2773 error = ENOENT;
2774 if (error == 0)
2775 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2776 return (error);
2777 }
2778
2779 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
2780 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2781 0, 0, tcp_getcred, "S,xucred",
2782 "Get the xucred of a TCP connection");
2783 #endif /* INET */
2784
2785 #ifdef INET6
2786 static int
tcp6_getcred(SYSCTL_HANDLER_ARGS)2787 tcp6_getcred(SYSCTL_HANDLER_ARGS)
2788 {
2789 struct epoch_tracker et;
2790 struct xucred xuc;
2791 struct sockaddr_in6 addrs[2];
2792 struct inpcb *inp;
2793 int error;
2794 #ifdef INET
2795 int mapped = 0;
2796 #endif
2797
2798 if (req->newptr == NULL)
2799 return (EINVAL);
2800 error = priv_check(req->td, PRIV_NETINET_GETCRED);
2801 if (error)
2802 return (error);
2803 error = SYSCTL_IN(req, addrs, sizeof(addrs));
2804 if (error)
2805 return (error);
2806 if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 ||
2807 (error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
2808 return (error);
2809 }
2810 if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
2811 #ifdef INET
2812 if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
2813 mapped = 1;
2814 else
2815 #endif
2816 return (EINVAL);
2817 }
2818
2819 NET_EPOCH_ENTER(et);
2820 #ifdef INET
2821 if (mapped == 1)
2822 inp = in_pcblookup(&V_tcbinfo,
2823 *(struct in_addr *)&addrs[1].sin6_addr.s6_addr[12],
2824 addrs[1].sin6_port,
2825 *(struct in_addr *)&addrs[0].sin6_addr.s6_addr[12],
2826 addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
2827 else
2828 #endif
2829 inp = in6_pcblookup(&V_tcbinfo,
2830 &addrs[1].sin6_addr, addrs[1].sin6_port,
2831 &addrs[0].sin6_addr, addrs[0].sin6_port,
2832 INPLOOKUP_RLOCKPCB, NULL);
2833 NET_EPOCH_EXIT(et);
2834 if (inp != NULL) {
2835 if (error == 0)
2836 error = cr_canseeinpcb(req->td->td_ucred, inp);
2837 if (error == 0)
2838 cru2x(inp->inp_cred, &xuc);
2839 INP_RUNLOCK(inp);
2840 } else
2841 error = ENOENT;
2842 if (error == 0)
2843 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2844 return (error);
2845 }
2846
2847 SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
2848 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2849 0, 0, tcp6_getcred, "S,xucred",
2850 "Get the xucred of a TCP6 connection");
2851 #endif /* INET6 */
2852
2853 #ifdef INET
2854 /* Path MTU to try next when a fragmentation-needed message is received. */
2855 static inline int
tcp_next_pmtu(const struct icmp * icp,const struct ip * ip)2856 tcp_next_pmtu(const struct icmp *icp, const struct ip *ip)
2857 {
2858 int mtu = ntohs(icp->icmp_nextmtu);
2859
2860 /* If no alternative MTU was proposed, try the next smaller one. */
2861 if (!mtu)
2862 mtu = ip_next_mtu(ntohs(ip->ip_len), 1);
2863 if (mtu < V_tcp_minmss + sizeof(struct tcpiphdr))
2864 mtu = V_tcp_minmss + sizeof(struct tcpiphdr);
2865
2866 return (mtu);
2867 }
2868
2869 static void
tcp_ctlinput_with_port(struct icmp * icp,uint16_t port)2870 tcp_ctlinput_with_port(struct icmp *icp, uint16_t port)
2871 {
2872 struct ip *ip;
2873 struct tcphdr *th;
2874 struct inpcb *inp;
2875 struct tcpcb *tp;
2876 struct inpcb *(*notify)(struct inpcb *, int);
2877 struct in_conninfo inc;
2878 tcp_seq icmp_tcp_seq;
2879 int errno, mtu;
2880
2881 errno = icmp_errmap(icp);
2882 switch (errno) {
2883 case 0:
2884 return;
2885 case EMSGSIZE:
2886 notify = tcp_mtudisc_notify;
2887 break;
2888 case ECONNREFUSED:
2889 if (V_icmp_may_rst)
2890 notify = tcp_drop_syn_sent;
2891 else
2892 notify = tcp_notify;
2893 break;
2894 case EHOSTUNREACH:
2895 if (V_icmp_may_rst && icp->icmp_type == ICMP_TIMXCEED)
2896 notify = tcp_drop_syn_sent;
2897 else
2898 notify = tcp_notify;
2899 break;
2900 default:
2901 notify = tcp_notify;
2902 }
2903
2904 ip = &icp->icmp_ip;
2905 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2906 icmp_tcp_seq = th->th_seq;
2907 inp = in_pcblookup(&V_tcbinfo, ip->ip_dst, th->th_dport, ip->ip_src,
2908 th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
2909 if (inp != NULL) {
2910 tp = intotcpcb(inp);
2911 #ifdef TCP_OFFLOAD
2912 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
2913 /*
2914 * MTU discovery for offloaded connections. Let
2915 * the TOE driver verify seq# and process it.
2916 */
2917 mtu = tcp_next_pmtu(icp, ip);
2918 tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
2919 goto out;
2920 }
2921 #endif
2922 if (tp->t_port != port)
2923 goto out;
2924 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
2925 SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
2926 if (errno == EMSGSIZE) {
2927 /*
2928 * MTU discovery: we got a needfrag and
2929 * will potentially try a lower MTU.
2930 */
2931 mtu = tcp_next_pmtu(icp, ip);
2932
2933 /*
2934 * Only process the offered MTU if it
2935 * is smaller than the current one.
2936 */
2937 if (mtu < tp->t_maxseg +
2938 sizeof(struct tcpiphdr)) {
2939 bzero(&inc, sizeof(inc));
2940 inc.inc_faddr = ip->ip_dst;
2941 inc.inc_fibnum =
2942 inp->inp_inc.inc_fibnum;
2943 tcp_hc_updatemtu(&inc, mtu);
2944 inp = tcp_mtudisc(inp, mtu);
2945 }
2946 } else
2947 inp = (*notify)(inp, errno);
2948 }
2949 } else {
2950 bzero(&inc, sizeof(inc));
2951 inc.inc_fport = th->th_dport;
2952 inc.inc_lport = th->th_sport;
2953 inc.inc_faddr = ip->ip_dst;
2954 inc.inc_laddr = ip->ip_src;
2955 syncache_unreach(&inc, icmp_tcp_seq, port);
2956 }
2957 out:
2958 if (inp != NULL)
2959 INP_WUNLOCK(inp);
2960 }
2961
2962 static void
tcp_ctlinput(struct icmp * icmp)2963 tcp_ctlinput(struct icmp *icmp)
2964 {
2965 tcp_ctlinput_with_port(icmp, htons(0));
2966 }
2967
2968 static void
tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)2969 tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)
2970 {
2971 /* Its a tunneled TCP over UDP icmp */
2972 struct icmp *icmp = param.icmp;
2973 struct ip *outer_ip, *inner_ip;
2974 struct udphdr *udp;
2975 struct tcphdr *th, ttemp;
2976 int i_hlen, o_len;
2977 uint16_t port;
2978
2979 outer_ip = (struct ip *)((caddr_t)icmp - sizeof(struct ip));
2980 inner_ip = &icmp->icmp_ip;
2981 i_hlen = inner_ip->ip_hl << 2;
2982 o_len = ntohs(outer_ip->ip_len);
2983 if (o_len <
2984 (sizeof(struct ip) + 8 + i_hlen + sizeof(struct udphdr) + offsetof(struct tcphdr, th_ack))) {
2985 /* Not enough data present */
2986 return;
2987 }
2988 /* Ok lets strip out the inner udphdr header by copying up on top of it the tcp hdr */
2989 udp = (struct udphdr *)(((caddr_t)inner_ip) + i_hlen);
2990 if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
2991 return;
2992 }
2993 port = udp->uh_dport;
2994 th = (struct tcphdr *)(udp + 1);
2995 memcpy(&ttemp, th, sizeof(struct tcphdr));
2996 memcpy(udp, &ttemp, sizeof(struct tcphdr));
2997 /* Now adjust down the size of the outer IP header */
2998 o_len -= sizeof(struct udphdr);
2999 outer_ip->ip_len = htons(o_len);
3000 /* Now call in to the normal handling code */
3001 tcp_ctlinput_with_port(icmp, port);
3002 }
3003 #endif /* INET */
3004
3005 #ifdef INET6
3006 static inline int
tcp6_next_pmtu(const struct icmp6_hdr * icmp6)3007 tcp6_next_pmtu(const struct icmp6_hdr *icmp6)
3008 {
3009 int mtu = ntohl(icmp6->icmp6_mtu);
3010
3011 /*
3012 * If no alternative MTU was proposed, or the proposed MTU was too
3013 * small, set to the min.
3014 */
3015 if (mtu < IPV6_MMTU)
3016 mtu = IPV6_MMTU - 8; /* XXXNP: what is the adjustment for? */
3017 return (mtu);
3018 }
3019
3020 static void
tcp6_ctlinput_with_port(struct ip6ctlparam * ip6cp,uint16_t port)3021 tcp6_ctlinput_with_port(struct ip6ctlparam *ip6cp, uint16_t port)
3022 {
3023 struct in6_addr *dst;
3024 struct inpcb *(*notify)(struct inpcb *, int);
3025 struct ip6_hdr *ip6;
3026 struct mbuf *m;
3027 struct inpcb *inp;
3028 struct tcpcb *tp;
3029 struct icmp6_hdr *icmp6;
3030 struct in_conninfo inc;
3031 struct tcp_ports {
3032 uint16_t th_sport;
3033 uint16_t th_dport;
3034 } t_ports;
3035 tcp_seq icmp_tcp_seq;
3036 unsigned int mtu;
3037 unsigned int off;
3038 int errno;
3039
3040 icmp6 = ip6cp->ip6c_icmp6;
3041 m = ip6cp->ip6c_m;
3042 ip6 = ip6cp->ip6c_ip6;
3043 off = ip6cp->ip6c_off;
3044 dst = &ip6cp->ip6c_finaldst->sin6_addr;
3045
3046 errno = icmp6_errmap(icmp6);
3047 switch (errno) {
3048 case 0:
3049 return;
3050 case EMSGSIZE:
3051 notify = tcp_mtudisc_notify;
3052 break;
3053 case ECONNREFUSED:
3054 if (V_icmp_may_rst)
3055 notify = tcp_drop_syn_sent;
3056 else
3057 notify = tcp_notify;
3058 break;
3059 case EHOSTUNREACH:
3060 /*
3061 * There are only four ICMPs that may reset connection:
3062 * - administratively prohibited
3063 * - port unreachable
3064 * - time exceeded in transit
3065 * - unknown next header
3066 */
3067 if (V_icmp_may_rst &&
3068 ((icmp6->icmp6_type == ICMP6_DST_UNREACH &&
3069 (icmp6->icmp6_code == ICMP6_DST_UNREACH_ADMIN ||
3070 icmp6->icmp6_code == ICMP6_DST_UNREACH_NOPORT)) ||
3071 (icmp6->icmp6_type == ICMP6_TIME_EXCEEDED &&
3072 icmp6->icmp6_code == ICMP6_TIME_EXCEED_TRANSIT) ||
3073 (icmp6->icmp6_type == ICMP6_PARAM_PROB &&
3074 icmp6->icmp6_code == ICMP6_PARAMPROB_NEXTHEADER)))
3075 notify = tcp_drop_syn_sent;
3076 else
3077 notify = tcp_notify;
3078 break;
3079 default:
3080 notify = tcp_notify;
3081 }
3082
3083 /* Check if we can safely get the ports from the tcp hdr */
3084 if (m == NULL ||
3085 (m->m_pkthdr.len <
3086 (int32_t) (off + sizeof(struct tcp_ports)))) {
3087 return;
3088 }
3089 bzero(&t_ports, sizeof(struct tcp_ports));
3090 m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports);
3091 inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport,
3092 &ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL);
3093 off += sizeof(struct tcp_ports);
3094 if (m->m_pkthdr.len < (int32_t) (off + sizeof(tcp_seq))) {
3095 goto out;
3096 }
3097 m_copydata(m, off, sizeof(tcp_seq), (caddr_t)&icmp_tcp_seq);
3098 if (inp != NULL) {
3099 tp = intotcpcb(inp);
3100 #ifdef TCP_OFFLOAD
3101 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
3102 /* MTU discovery for offloaded connections. */
3103 mtu = tcp6_next_pmtu(icmp6);
3104 tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
3105 goto out;
3106 }
3107 #endif
3108 if (tp->t_port != port)
3109 goto out;
3110 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
3111 SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
3112 if (errno == EMSGSIZE) {
3113 /*
3114 * MTU discovery:
3115 * If we got a needfrag set the MTU
3116 * in the route to the suggested new
3117 * value (if given) and then notify.
3118 */
3119 mtu = tcp6_next_pmtu(icmp6);
3120
3121 bzero(&inc, sizeof(inc));
3122 inc.inc_fibnum = M_GETFIB(m);
3123 inc.inc_flags |= INC_ISIPV6;
3124 inc.inc6_faddr = *dst;
3125 if (in6_setscope(&inc.inc6_faddr,
3126 m->m_pkthdr.rcvif, NULL))
3127 goto out;
3128 /*
3129 * Only process the offered MTU if it
3130 * is smaller than the current one.
3131 */
3132 if (mtu < tp->t_maxseg +
3133 sizeof (struct tcphdr) +
3134 sizeof (struct ip6_hdr)) {
3135 tcp_hc_updatemtu(&inc, mtu);
3136 tcp_mtudisc(inp, mtu);
3137 ICMP6STAT_INC(icp6s_pmtuchg);
3138 }
3139 } else
3140 inp = (*notify)(inp, errno);
3141 }
3142 } else {
3143 bzero(&inc, sizeof(inc));
3144 inc.inc_fibnum = M_GETFIB(m);
3145 inc.inc_flags |= INC_ISIPV6;
3146 inc.inc_fport = t_ports.th_dport;
3147 inc.inc_lport = t_ports.th_sport;
3148 inc.inc6_faddr = *dst;
3149 inc.inc6_laddr = ip6->ip6_src;
3150 syncache_unreach(&inc, icmp_tcp_seq, port);
3151 }
3152 out:
3153 if (inp != NULL)
3154 INP_WUNLOCK(inp);
3155 }
3156
3157 static void
tcp6_ctlinput(struct ip6ctlparam * ctl)3158 tcp6_ctlinput(struct ip6ctlparam *ctl)
3159 {
3160 tcp6_ctlinput_with_port(ctl, htons(0));
3161 }
3162
3163 static void
tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)3164 tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)
3165 {
3166 struct ip6ctlparam *ip6cp = param.ip6cp;
3167 struct mbuf *m;
3168 struct udphdr *udp;
3169 uint16_t port;
3170
3171 m = m_pulldown(ip6cp->ip6c_m, ip6cp->ip6c_off, sizeof(struct udphdr), NULL);
3172 if (m == NULL) {
3173 return;
3174 }
3175 udp = mtod(m, struct udphdr *);
3176 if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
3177 return;
3178 }
3179 port = udp->uh_dport;
3180 m_adj(m, sizeof(struct udphdr));
3181 if ((m->m_flags & M_PKTHDR) == 0) {
3182 ip6cp->ip6c_m->m_pkthdr.len -= sizeof(struct udphdr);
3183 }
3184 /* Now call in to the normal handling code */
3185 tcp6_ctlinput_with_port(ip6cp, port);
3186 }
3187
3188 #endif /* INET6 */
3189
3190 static uint32_t
tcp_keyed_hash(struct in_conninfo * inc,u_char * key,u_int len)3191 tcp_keyed_hash(struct in_conninfo *inc, u_char *key, u_int len)
3192 {
3193 SIPHASH_CTX ctx;
3194 uint32_t hash[2];
3195
3196 KASSERT(len >= SIPHASH_KEY_LENGTH,
3197 ("%s: keylen %u too short ", __func__, len));
3198 SipHash24_Init(&ctx);
3199 SipHash_SetKey(&ctx, (uint8_t *)key);
3200 SipHash_Update(&ctx, &inc->inc_fport, sizeof(uint16_t));
3201 SipHash_Update(&ctx, &inc->inc_lport, sizeof(uint16_t));
3202 switch (inc->inc_flags & INC_ISIPV6) {
3203 #ifdef INET
3204 case 0:
3205 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(struct in_addr));
3206 SipHash_Update(&ctx, &inc->inc_laddr, sizeof(struct in_addr));
3207 break;
3208 #endif
3209 #ifdef INET6
3210 case INC_ISIPV6:
3211 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(struct in6_addr));
3212 SipHash_Update(&ctx, &inc->inc6_laddr, sizeof(struct in6_addr));
3213 break;
3214 #endif
3215 }
3216 SipHash_Final((uint8_t *)hash, &ctx);
3217
3218 return (hash[0] ^ hash[1]);
3219 }
3220
3221 uint32_t
tcp_new_ts_offset(struct in_conninfo * inc)3222 tcp_new_ts_offset(struct in_conninfo *inc)
3223 {
3224 struct in_conninfo inc_store, *local_inc;
3225
3226 if (!V_tcp_ts_offset_per_conn) {
3227 memcpy(&inc_store, inc, sizeof(struct in_conninfo));
3228 inc_store.inc_lport = 0;
3229 inc_store.inc_fport = 0;
3230 local_inc = &inc_store;
3231 } else {
3232 local_inc = inc;
3233 }
3234 return (tcp_keyed_hash(local_inc, V_ts_offset_secret,
3235 sizeof(V_ts_offset_secret)));
3236 }
3237
3238 /*
3239 * Following is where TCP initial sequence number generation occurs.
3240 *
3241 * There are two places where we must use initial sequence numbers:
3242 * 1. In SYN-ACK packets.
3243 * 2. In SYN packets.
3244 *
3245 * All ISNs for SYN-ACK packets are generated by the syncache. See
3246 * tcp_syncache.c for details.
3247 *
3248 * The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
3249 * depends on this property. In addition, these ISNs should be
3250 * unguessable so as to prevent connection hijacking. To satisfy
3251 * the requirements of this situation, the algorithm outlined in
3252 * RFC 1948 is used, with only small modifications.
3253 *
3254 * Implementation details:
3255 *
3256 * Time is based off the system timer, and is corrected so that it
3257 * increases by one megabyte per second. This allows for proper
3258 * recycling on high speed LANs while still leaving over an hour
3259 * before rollover.
3260 *
3261 * As reading the *exact* system time is too expensive to be done
3262 * whenever setting up a TCP connection, we increment the time
3263 * offset in two ways. First, a small random positive increment
3264 * is added to isn_offset for each connection that is set up.
3265 * Second, the function tcp_isn_tick fires once per clock tick
3266 * and increments isn_offset as necessary so that sequence numbers
3267 * are incremented at approximately ISN_BYTES_PER_SECOND. The
3268 * random positive increments serve only to ensure that the same
3269 * exact sequence number is never sent out twice (as could otherwise
3270 * happen when a port is recycled in less than the system tick
3271 * interval.)
3272 *
3273 * net.inet.tcp.isn_reseed_interval controls the number of seconds
3274 * between seeding of isn_secret. This is normally set to zero,
3275 * as reseeding should not be necessary.
3276 *
3277 * Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
3278 * isn_offset_old, and isn_ctx is performed using the ISN lock. In
3279 * general, this means holding an exclusive (write) lock.
3280 */
3281
3282 #define ISN_BYTES_PER_SECOND 1048576
3283 #define ISN_STATIC_INCREMENT 4096
3284 #define ISN_RANDOM_INCREMENT (4096 - 1)
3285 #define ISN_SECRET_LENGTH SIPHASH_KEY_LENGTH
3286
3287 VNET_DEFINE_STATIC(u_char, isn_secret[ISN_SECRET_LENGTH]);
3288 VNET_DEFINE_STATIC(int, isn_last);
3289 VNET_DEFINE_STATIC(int, isn_last_reseed);
3290 VNET_DEFINE_STATIC(u_int32_t, isn_offset);
3291 VNET_DEFINE_STATIC(u_int32_t, isn_offset_old);
3292
3293 #define V_isn_secret VNET(isn_secret)
3294 #define V_isn_last VNET(isn_last)
3295 #define V_isn_last_reseed VNET(isn_last_reseed)
3296 #define V_isn_offset VNET(isn_offset)
3297 #define V_isn_offset_old VNET(isn_offset_old)
3298
3299 tcp_seq
tcp_new_isn(struct in_conninfo * inc)3300 tcp_new_isn(struct in_conninfo *inc)
3301 {
3302 tcp_seq new_isn;
3303 u_int32_t projected_offset;
3304
3305 ISN_LOCK();
3306 /* Seed if this is the first use, reseed if requested. */
3307 if ((V_isn_last_reseed == 0) || ((V_tcp_isn_reseed_interval > 0) &&
3308 (((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
3309 < (u_int)ticks))) {
3310 arc4rand(&V_isn_secret, sizeof(V_isn_secret), 0);
3311 V_isn_last_reseed = ticks;
3312 }
3313
3314 /* Compute the hash and return the ISN. */
3315 new_isn = (tcp_seq)tcp_keyed_hash(inc, V_isn_secret,
3316 sizeof(V_isn_secret));
3317 V_isn_offset += ISN_STATIC_INCREMENT +
3318 (arc4random() & ISN_RANDOM_INCREMENT);
3319 if (ticks != V_isn_last) {
3320 projected_offset = V_isn_offset_old +
3321 ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
3322 if (SEQ_GT(projected_offset, V_isn_offset))
3323 V_isn_offset = projected_offset;
3324 V_isn_offset_old = V_isn_offset;
3325 V_isn_last = ticks;
3326 }
3327 new_isn += V_isn_offset;
3328 ISN_UNLOCK();
3329 return (new_isn);
3330 }
3331
3332 /*
3333 * When a specific ICMP unreachable message is received and the
3334 * connection state is SYN-SENT, drop the connection. This behavior
3335 * is controlled by the icmp_may_rst sysctl.
3336 */
3337 static struct inpcb *
tcp_drop_syn_sent(struct inpcb * inp,int errno)3338 tcp_drop_syn_sent(struct inpcb *inp, int errno)
3339 {
3340 struct tcpcb *tp;
3341
3342 NET_EPOCH_ASSERT();
3343 INP_WLOCK_ASSERT(inp);
3344
3345 tp = intotcpcb(inp);
3346 if (tp->t_state != TCPS_SYN_SENT)
3347 return (inp);
3348
3349 if (IS_FASTOPEN(tp->t_flags))
3350 tcp_fastopen_disable_path(tp);
3351
3352 tp = tcp_drop(tp, errno);
3353 if (tp != NULL)
3354 return (inp);
3355 else
3356 return (NULL);
3357 }
3358
3359 /*
3360 * When `need fragmentation' ICMP is received, update our idea of the MSS
3361 * based on the new value. Also nudge TCP to send something, since we
3362 * know the packet we just sent was dropped.
3363 * This duplicates some code in the tcp_mss() function in tcp_input.c.
3364 */
3365 static struct inpcb *
tcp_mtudisc_notify(struct inpcb * inp,int error)3366 tcp_mtudisc_notify(struct inpcb *inp, int error)
3367 {
3368
3369 return (tcp_mtudisc(inp, -1));
3370 }
3371
3372 static struct inpcb *
tcp_mtudisc(struct inpcb * inp,int mtuoffer)3373 tcp_mtudisc(struct inpcb *inp, int mtuoffer)
3374 {
3375 struct tcpcb *tp;
3376 struct socket *so;
3377
3378 INP_WLOCK_ASSERT(inp);
3379
3380 tp = intotcpcb(inp);
3381 KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
3382
3383 tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
3384
3385 so = inp->inp_socket;
3386 SOCKBUF_LOCK(&so->so_snd);
3387 /* If the mss is larger than the socket buffer, decrease the mss. */
3388 if (so->so_snd.sb_hiwat < tp->t_maxseg)
3389 tp->t_maxseg = so->so_snd.sb_hiwat;
3390 SOCKBUF_UNLOCK(&so->so_snd);
3391
3392 TCPSTAT_INC(tcps_mturesent);
3393 tp->t_rtttime = 0;
3394 tp->snd_nxt = tp->snd_una;
3395 tcp_free_sackholes(tp);
3396 tp->snd_recover = tp->snd_max;
3397 if (tp->t_flags & TF_SACK_PERMIT)
3398 EXIT_FASTRECOVERY(tp->t_flags);
3399 if (tp->t_fb->tfb_tcp_mtu_chg != NULL) {
3400 /*
3401 * Conceptually the snd_nxt setting
3402 * and freeing sack holes should
3403 * be done by the default stacks
3404 * own tfb_tcp_mtu_chg().
3405 */
3406 tp->t_fb->tfb_tcp_mtu_chg(tp);
3407 }
3408 if (tcp_output(tp) < 0)
3409 return (NULL);
3410 else
3411 return (inp);
3412 }
3413
3414 #ifdef INET
3415 /*
3416 * Look-up the routing entry to the peer of this inpcb. If no route
3417 * is found and it cannot be allocated, then return 0. This routine
3418 * is called by TCP routines that access the rmx structure and by
3419 * tcp_mss_update to get the peer/interface MTU.
3420 */
3421 uint32_t
tcp_maxmtu(struct in_conninfo * inc,struct tcp_ifcap * cap)3422 tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
3423 {
3424 struct nhop_object *nh;
3425 struct ifnet *ifp;
3426 uint32_t maxmtu = 0;
3427
3428 KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));
3429
3430 if (inc->inc_faddr.s_addr != INADDR_ANY) {
3431 nh = fib4_lookup(inc->inc_fibnum, inc->inc_faddr, 0, NHR_NONE, 0);
3432 if (nh == NULL)
3433 return (0);
3434
3435 ifp = nh->nh_ifp;
3436 maxmtu = nh->nh_mtu;
3437
3438 /* Report additional interface capabilities. */
3439 if (cap != NULL) {
3440 if (ifp->if_capenable & IFCAP_TSO4 &&
3441 ifp->if_hwassist & CSUM_TSO) {
3442 cap->ifcap |= CSUM_TSO;
3443 cap->tsomax = ifp->if_hw_tsomax;
3444 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3445 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3446 }
3447 }
3448 }
3449 return (maxmtu);
3450 }
3451 #endif /* INET */
3452
3453 #ifdef INET6
3454 uint32_t
tcp_maxmtu6(struct in_conninfo * inc,struct tcp_ifcap * cap)3455 tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
3456 {
3457 struct nhop_object *nh;
3458 struct in6_addr dst6;
3459 uint32_t scopeid;
3460 struct ifnet *ifp;
3461 uint32_t maxmtu = 0;
3462
3463 KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));
3464
3465 if (inc->inc_flags & INC_IPV6MINMTU)
3466 return (IPV6_MMTU);
3467
3468 if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
3469 in6_splitscope(&inc->inc6_faddr, &dst6, &scopeid);
3470 nh = fib6_lookup(inc->inc_fibnum, &dst6, scopeid, NHR_NONE, 0);
3471 if (nh == NULL)
3472 return (0);
3473
3474 ifp = nh->nh_ifp;
3475 maxmtu = nh->nh_mtu;
3476
3477 /* Report additional interface capabilities. */
3478 if (cap != NULL) {
3479 if (ifp->if_capenable & IFCAP_TSO6 &&
3480 ifp->if_hwassist & CSUM_TSO) {
3481 cap->ifcap |= CSUM_TSO;
3482 cap->tsomax = ifp->if_hw_tsomax;
3483 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3484 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3485 }
3486 }
3487 }
3488
3489 return (maxmtu);
3490 }
3491
3492 /*
3493 * Handle setsockopt(IPV6_USE_MIN_MTU) by a TCP stack.
3494 *
3495 * XXXGL: we are updating inpcb here with INC_IPV6MINMTU flag.
3496 * The right place to do that is ip6_setpktopt() that has just been
3497 * executed. By the way it just filled ip6po_minmtu for us.
3498 */
3499 void
tcp6_use_min_mtu(struct tcpcb * tp)3500 tcp6_use_min_mtu(struct tcpcb *tp)
3501 {
3502 struct inpcb *inp = tptoinpcb(tp);
3503
3504 INP_WLOCK_ASSERT(inp);
3505 /*
3506 * In case of the IPV6_USE_MIN_MTU socket
3507 * option, the INC_IPV6MINMTU flag to announce
3508 * a corresponding MSS during the initial
3509 * handshake. If the TCP connection is not in
3510 * the front states, just reduce the MSS being
3511 * used. This avoids the sending of TCP
3512 * segments which will be fragmented at the
3513 * IPv6 layer.
3514 */
3515 inp->inp_inc.inc_flags |= INC_IPV6MINMTU;
3516 if ((tp->t_state >= TCPS_SYN_SENT) &&
3517 (inp->inp_inc.inc_flags & INC_ISIPV6)) {
3518 struct ip6_pktopts *opt;
3519
3520 opt = inp->in6p_outputopts;
3521 if (opt != NULL && opt->ip6po_minmtu == IP6PO_MINMTU_ALL &&
3522 tp->t_maxseg > TCP6_MSS)
3523 tp->t_maxseg = TCP6_MSS;
3524 }
3525 }
3526 #endif /* INET6 */
3527
3528 /*
3529 * Calculate effective SMSS per RFC5681 definition for a given TCP
3530 * connection at its current state, taking into account SACK and etc.
3531 */
3532 u_int
tcp_maxseg(const struct tcpcb * tp)3533 tcp_maxseg(const struct tcpcb *tp)
3534 {
3535 u_int optlen;
3536
3537 if (tp->t_flags & TF_NOOPT)
3538 return (tp->t_maxseg);
3539
3540 /*
3541 * Here we have a simplified code from tcp_addoptions(),
3542 * without a proper loop, and having most of paddings hardcoded.
3543 * We might make mistakes with padding here in some edge cases,
3544 * but this is harmless, since result of tcp_maxseg() is used
3545 * only in cwnd and ssthresh estimations.
3546 */
3547 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3548 if (tp->t_flags & TF_RCVD_TSTMP)
3549 optlen = TCPOLEN_TSTAMP_APPA;
3550 else
3551 optlen = 0;
3552 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3553 if (tp->t_flags & TF_SIGNATURE)
3554 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3555 #endif
3556 if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) {
3557 optlen += TCPOLEN_SACKHDR;
3558 optlen += tp->rcv_numsacks * TCPOLEN_SACK;
3559 optlen = PADTCPOLEN(optlen);
3560 }
3561 } else {
3562 if (tp->t_flags & TF_REQ_TSTMP)
3563 optlen = TCPOLEN_TSTAMP_APPA;
3564 else
3565 optlen = PADTCPOLEN(TCPOLEN_MAXSEG);
3566 if (tp->t_flags & TF_REQ_SCALE)
3567 optlen += PADTCPOLEN(TCPOLEN_WINDOW);
3568 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3569 if (tp->t_flags & TF_SIGNATURE)
3570 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3571 #endif
3572 if (tp->t_flags & TF_SACK_PERMIT)
3573 optlen += PADTCPOLEN(TCPOLEN_SACK_PERMITTED);
3574 }
3575 #undef PAD
3576 optlen = min(optlen, TCP_MAXOLEN);
3577 return (tp->t_maxseg - optlen);
3578 }
3579
3580
3581 u_int
tcp_fixed_maxseg(const struct tcpcb * tp)3582 tcp_fixed_maxseg(const struct tcpcb *tp)
3583 {
3584 int optlen;
3585
3586 if (tp->t_flags & TF_NOOPT)
3587 return (tp->t_maxseg);
3588
3589 /*
3590 * Here we have a simplified code from tcp_addoptions(),
3591 * without a proper loop, and having most of paddings hardcoded.
3592 * We only consider fixed options that we would send every
3593 * time I.e. SACK is not considered. This is important
3594 * for cc modules to figure out what the modulo of the
3595 * cwnd should be.
3596 */
3597 #define PAD(len) ((((len) / 4) + !!((len) % 4)) * 4)
3598 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3599 if (tp->t_flags & TF_RCVD_TSTMP)
3600 optlen = TCPOLEN_TSTAMP_APPA;
3601 else
3602 optlen = 0;
3603 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3604 if (tp->t_flags & TF_SIGNATURE)
3605 optlen += PAD(TCPOLEN_SIGNATURE);
3606 #endif
3607 } else {
3608 if (tp->t_flags & TF_REQ_TSTMP)
3609 optlen = TCPOLEN_TSTAMP_APPA;
3610 else
3611 optlen = PAD(TCPOLEN_MAXSEG);
3612 if (tp->t_flags & TF_REQ_SCALE)
3613 optlen += PAD(TCPOLEN_WINDOW);
3614 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3615 if (tp->t_flags & TF_SIGNATURE)
3616 optlen += PAD(TCPOLEN_SIGNATURE);
3617 #endif
3618 if (tp->t_flags & TF_SACK_PERMIT)
3619 optlen += PAD(TCPOLEN_SACK_PERMITTED);
3620 }
3621 #undef PAD
3622 optlen = min(optlen, TCP_MAXOLEN);
3623 return (tp->t_maxseg - optlen);
3624 }
3625
3626
3627
3628 static int
sysctl_drop(SYSCTL_HANDLER_ARGS)3629 sysctl_drop(SYSCTL_HANDLER_ARGS)
3630 {
3631 /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3632 struct sockaddr_storage addrs[2];
3633 struct inpcb *inp;
3634 struct tcpcb *tp;
3635 #ifdef INET
3636 struct sockaddr_in *fin = NULL, *lin = NULL;
3637 #endif
3638 struct epoch_tracker et;
3639 #ifdef INET6
3640 struct sockaddr_in6 *fin6, *lin6;
3641 #endif
3642 int error;
3643
3644 inp = NULL;
3645 #ifdef INET6
3646 fin6 = lin6 = NULL;
3647 #endif
3648 error = 0;
3649
3650 if (req->oldptr != NULL || req->oldlen != 0)
3651 return (EINVAL);
3652 if (req->newptr == NULL)
3653 return (EPERM);
3654 if (req->newlen < sizeof(addrs))
3655 return (ENOMEM);
3656 error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3657 if (error)
3658 return (error);
3659
3660 switch (addrs[0].ss_family) {
3661 #ifdef INET6
3662 case AF_INET6:
3663 fin6 = (struct sockaddr_in6 *)&addrs[0];
3664 lin6 = (struct sockaddr_in6 *)&addrs[1];
3665 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3666 lin6->sin6_len != sizeof(struct sockaddr_in6))
3667 return (EINVAL);
3668 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3669 if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3670 return (EINVAL);
3671 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3672 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3673 #ifdef INET
3674 fin = (struct sockaddr_in *)&addrs[0];
3675 lin = (struct sockaddr_in *)&addrs[1];
3676 #endif
3677 break;
3678 }
3679 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3680 if (error)
3681 return (error);
3682 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3683 if (error)
3684 return (error);
3685 break;
3686 #endif
3687 #ifdef INET
3688 case AF_INET:
3689 fin = (struct sockaddr_in *)&addrs[0];
3690 lin = (struct sockaddr_in *)&addrs[1];
3691 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3692 lin->sin_len != sizeof(struct sockaddr_in))
3693 return (EINVAL);
3694 break;
3695 #endif
3696 default:
3697 return (EINVAL);
3698 }
3699 NET_EPOCH_ENTER(et);
3700 switch (addrs[0].ss_family) {
3701 #ifdef INET6
3702 case AF_INET6:
3703 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3704 fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3705 INPLOOKUP_WLOCKPCB, NULL);
3706 break;
3707 #endif
3708 #ifdef INET
3709 case AF_INET:
3710 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3711 lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3712 break;
3713 #endif
3714 }
3715 if (inp != NULL) {
3716 if (!SOLISTENING(inp->inp_socket)) {
3717 tp = intotcpcb(inp);
3718 tp = tcp_drop(tp, ECONNABORTED);
3719 if (tp != NULL)
3720 INP_WUNLOCK(inp);
3721 } else
3722 INP_WUNLOCK(inp);
3723 } else
3724 error = ESRCH;
3725 NET_EPOCH_EXIT(et);
3726 return (error);
3727 }
3728
3729 SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
3730 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3731 CTLFLAG_NEEDGIANT, NULL, 0, sysctl_drop, "",
3732 "Drop TCP connection");
3733
3734 static int
tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)3735 tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)
3736 {
3737 return (sysctl_setsockopt(oidp, arg1, arg2, req, &V_tcbinfo,
3738 &tcp_ctloutput_set));
3739 }
3740
3741 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, setsockopt,
3742 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3743 CTLFLAG_MPSAFE, NULL, 0, tcp_sysctl_setsockopt, "",
3744 "Set socket option for TCP endpoint");
3745
3746 #ifdef KERN_TLS
3747 static int
sysctl_switch_tls(SYSCTL_HANDLER_ARGS)3748 sysctl_switch_tls(SYSCTL_HANDLER_ARGS)
3749 {
3750 /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3751 struct sockaddr_storage addrs[2];
3752 struct inpcb *inp;
3753 #ifdef INET
3754 struct sockaddr_in *fin = NULL, *lin = NULL;
3755 #endif
3756 struct epoch_tracker et;
3757 #ifdef INET6
3758 struct sockaddr_in6 *fin6, *lin6;
3759 #endif
3760 int error;
3761
3762 inp = NULL;
3763 #ifdef INET6
3764 fin6 = lin6 = NULL;
3765 #endif
3766 error = 0;
3767
3768 if (req->oldptr != NULL || req->oldlen != 0)
3769 return (EINVAL);
3770 if (req->newptr == NULL)
3771 return (EPERM);
3772 if (req->newlen < sizeof(addrs))
3773 return (ENOMEM);
3774 error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3775 if (error)
3776 return (error);
3777
3778 switch (addrs[0].ss_family) {
3779 #ifdef INET6
3780 case AF_INET6:
3781 fin6 = (struct sockaddr_in6 *)&addrs[0];
3782 lin6 = (struct sockaddr_in6 *)&addrs[1];
3783 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3784 lin6->sin6_len != sizeof(struct sockaddr_in6))
3785 return (EINVAL);
3786 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3787 if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3788 return (EINVAL);
3789 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3790 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3791 #ifdef INET
3792 fin = (struct sockaddr_in *)&addrs[0];
3793 lin = (struct sockaddr_in *)&addrs[1];
3794 #endif
3795 break;
3796 }
3797 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3798 if (error)
3799 return (error);
3800 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3801 if (error)
3802 return (error);
3803 break;
3804 #endif
3805 #ifdef INET
3806 case AF_INET:
3807 fin = (struct sockaddr_in *)&addrs[0];
3808 lin = (struct sockaddr_in *)&addrs[1];
3809 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3810 lin->sin_len != sizeof(struct sockaddr_in))
3811 return (EINVAL);
3812 break;
3813 #endif
3814 default:
3815 return (EINVAL);
3816 }
3817 NET_EPOCH_ENTER(et);
3818 switch (addrs[0].ss_family) {
3819 #ifdef INET6
3820 case AF_INET6:
3821 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3822 fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3823 INPLOOKUP_WLOCKPCB, NULL);
3824 break;
3825 #endif
3826 #ifdef INET
3827 case AF_INET:
3828 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3829 lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3830 break;
3831 #endif
3832 }
3833 NET_EPOCH_EXIT(et);
3834 if (inp != NULL) {
3835 struct socket *so;
3836
3837 so = inp->inp_socket;
3838 soref(so);
3839 error = ktls_set_tx_mode(so,
3840 arg2 == 0 ? TCP_TLS_MODE_SW : TCP_TLS_MODE_IFNET);
3841 INP_WUNLOCK(inp);
3842 sorele(so);
3843 } else
3844 error = ESRCH;
3845 return (error);
3846 }
3847
3848 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_sw_tls,
3849 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3850 CTLFLAG_NEEDGIANT, NULL, 0, sysctl_switch_tls, "",
3851 "Switch TCP connection to SW TLS");
3852 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_ifnet_tls,
3853 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3854 CTLFLAG_NEEDGIANT, NULL, 1, sysctl_switch_tls, "",
3855 "Switch TCP connection to ifnet TLS");
3856 #endif
3857
3858 /*
3859 * Generate a standardized TCP log line for use throughout the
3860 * tcp subsystem. Memory allocation is done with M_NOWAIT to
3861 * allow use in the interrupt context.
3862 *
3863 * NB: The caller MUST free(s, M_TCPLOG) the returned string.
3864 * NB: The function may return NULL if memory allocation failed.
3865 *
3866 * Due to header inclusion and ordering limitations the struct ip
3867 * and ip6_hdr pointers have to be passed as void pointers.
3868 */
3869 char *
tcp_log_vain(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3870 tcp_log_vain(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3871 const void *ip6hdr)
3872 {
3873
3874 /* Is logging enabled? */
3875 if (V_tcp_log_in_vain == 0)
3876 return (NULL);
3877
3878 return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3879 }
3880
3881 char *
tcp_log_addrs(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3882 tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3883 const void *ip6hdr)
3884 {
3885
3886 /* Is logging enabled? */
3887 if (tcp_log_debug == 0)
3888 return (NULL);
3889
3890 return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3891 }
3892
3893 static char *
tcp_log_addr(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3894 tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3895 const void *ip6hdr)
3896 {
3897 char *s, *sp;
3898 size_t size;
3899 #ifdef INET
3900 const struct ip *ip = (const struct ip *)ip4hdr;
3901 #endif
3902 #ifdef INET6
3903 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)ip6hdr;
3904 #endif /* INET6 */
3905
3906 /*
3907 * The log line looks like this:
3908 * "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
3909 */
3910 size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
3911 sizeof(PRINT_TH_FLAGS) + 1 +
3912 #ifdef INET6
3913 2 * INET6_ADDRSTRLEN;
3914 #else
3915 2 * INET_ADDRSTRLEN;
3916 #endif /* INET6 */
3917
3918 s = malloc(size, M_TCPLOG, M_ZERO|M_NOWAIT);
3919 if (s == NULL)
3920 return (NULL);
3921
3922 strcat(s, "TCP: [");
3923 sp = s + strlen(s);
3924
3925 if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
3926 inet_ntoa_r(inc->inc_faddr, sp);
3927 sp = s + strlen(s);
3928 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3929 sp = s + strlen(s);
3930 inet_ntoa_r(inc->inc_laddr, sp);
3931 sp = s + strlen(s);
3932 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3933 #ifdef INET6
3934 } else if (inc) {
3935 ip6_sprintf(sp, &inc->inc6_faddr);
3936 sp = s + strlen(s);
3937 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3938 sp = s + strlen(s);
3939 ip6_sprintf(sp, &inc->inc6_laddr);
3940 sp = s + strlen(s);
3941 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3942 } else if (ip6 && th) {
3943 ip6_sprintf(sp, &ip6->ip6_src);
3944 sp = s + strlen(s);
3945 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3946 sp = s + strlen(s);
3947 ip6_sprintf(sp, &ip6->ip6_dst);
3948 sp = s + strlen(s);
3949 sprintf(sp, "]:%i", ntohs(th->th_dport));
3950 #endif /* INET6 */
3951 #ifdef INET
3952 } else if (ip && th) {
3953 inet_ntoa_r(ip->ip_src, sp);
3954 sp = s + strlen(s);
3955 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3956 sp = s + strlen(s);
3957 inet_ntoa_r(ip->ip_dst, sp);
3958 sp = s + strlen(s);
3959 sprintf(sp, "]:%i", ntohs(th->th_dport));
3960 #endif /* INET */
3961 } else {
3962 free(s, M_TCPLOG);
3963 return (NULL);
3964 }
3965 sp = s + strlen(s);
3966 if (th)
3967 sprintf(sp, " tcpflags 0x%b", tcp_get_flags(th), PRINT_TH_FLAGS);
3968 if (*(s + size - 1) != '\0')
3969 panic("%s: string too long", __func__);
3970 return (s);
3971 }
3972
3973 /*
3974 * A subroutine which makes it easy to track TCP state changes with DTrace.
3975 * This function shouldn't be called for t_state initializations that don't
3976 * correspond to actual TCP state transitions.
3977 */
3978 void
tcp_state_change(struct tcpcb * tp,int newstate)3979 tcp_state_change(struct tcpcb *tp, int newstate)
3980 {
3981 #if defined(KDTRACE_HOOKS)
3982 int pstate = tp->t_state;
3983 #endif
3984
3985 TCPSTATES_DEC(tp->t_state);
3986 TCPSTATES_INC(newstate);
3987 tp->t_state = newstate;
3988 TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
3989 }
3990
3991 /*
3992 * Create an external-format (``xtcpcb'') structure using the information in
3993 * the kernel-format tcpcb structure pointed to by tp. This is done to
3994 * reduce the spew of irrelevant information over this interface, to isolate
3995 * user code from changes in the kernel structure, and potentially to provide
3996 * information-hiding if we decide that some of this information should be
3997 * hidden from users.
3998 */
3999 void
tcp_inptoxtp(const struct inpcb * inp,struct xtcpcb * xt)4000 tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt)
4001 {
4002 struct tcpcb *tp = intotcpcb(inp);
4003 sbintime_t now;
4004
4005 bzero(xt, sizeof(*xt));
4006 xt->t_state = tp->t_state;
4007 xt->t_logstate = tcp_get_bblog_state(tp);
4008 xt->t_flags = tp->t_flags;
4009 xt->t_sndzerowin = tp->t_sndzerowin;
4010 xt->t_sndrexmitpack = tp->t_sndrexmitpack;
4011 xt->t_rcvoopack = tp->t_rcvoopack;
4012 xt->t_rcv_wnd = tp->rcv_wnd;
4013 xt->t_snd_wnd = tp->snd_wnd;
4014 xt->t_snd_cwnd = tp->snd_cwnd;
4015 xt->t_snd_ssthresh = tp->snd_ssthresh;
4016 xt->t_dsack_bytes = tp->t_dsack_bytes;
4017 xt->t_dsack_tlp_bytes = tp->t_dsack_tlp_bytes;
4018 xt->t_dsack_pack = tp->t_dsack_pack;
4019 xt->t_maxseg = tp->t_maxseg;
4020 xt->xt_ecn = (tp->t_flags2 & TF2_ECN_PERMIT) ? 1 : 0 +
4021 (tp->t_flags2 & TF2_ACE_PERMIT) ? 2 : 0;
4022
4023 now = getsbinuptime();
4024 #define COPYTIMER(which,where) do { \
4025 if (tp->t_timers[which] != SBT_MAX) \
4026 xt->where = (tp->t_timers[which] - now) / SBT_1MS; \
4027 else \
4028 xt->where = 0; \
4029 } while (0)
4030 COPYTIMER(TT_DELACK, tt_delack);
4031 COPYTIMER(TT_REXMT, tt_rexmt);
4032 COPYTIMER(TT_PERSIST, tt_persist);
4033 COPYTIMER(TT_KEEP, tt_keep);
4034 COPYTIMER(TT_2MSL, tt_2msl);
4035 #undef COPYTIMER
4036 xt->t_rcvtime = 1000 * (ticks - tp->t_rcvtime) / hz;
4037
4038 xt->xt_encaps_port = tp->t_port;
4039 bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack,
4040 TCP_FUNCTION_NAME_LEN_MAX);
4041 bcopy(CC_ALGO(tp)->name, xt->xt_cc, TCP_CA_NAME_MAX);
4042 #ifdef TCP_BLACKBOX
4043 (void)tcp_log_get_id(tp, xt->xt_logid);
4044 #endif
4045
4046 xt->xt_len = sizeof(struct xtcpcb);
4047 in_pcbtoxinpcb(inp, &xt->xt_inp);
4048 /*
4049 * TCP doesn't use inp_ppcb pointer, we embed inpcb into tcpcb.
4050 * Fixup the pointer that in_pcbtoxinpcb() has set. When printing
4051 * TCP netstat(1) used to use this pointer, so this fixup needs to
4052 * stay for stable/14.
4053 */
4054 xt->xt_inp.inp_ppcb = (uintptr_t)tp;
4055 }
4056
4057 void
tcp_log_end_status(struct tcpcb * tp,uint8_t status)4058 tcp_log_end_status(struct tcpcb *tp, uint8_t status)
4059 {
4060 uint32_t bit, i;
4061
4062 if ((tp == NULL) ||
4063 (status > TCP_EI_STATUS_MAX_VALUE) ||
4064 (status == 0)) {
4065 /* Invalid */
4066 return;
4067 }
4068 if (status > (sizeof(uint32_t) * 8)) {
4069 /* Should this be a KASSERT? */
4070 return;
4071 }
4072 bit = 1U << (status - 1);
4073 if (bit & tp->t_end_info_status) {
4074 /* already logged */
4075 return;
4076 }
4077 for (i = 0; i < TCP_END_BYTE_INFO; i++) {
4078 if (tp->t_end_info_bytes[i] == TCP_EI_EMPTY_SLOT) {
4079 tp->t_end_info_bytes[i] = status;
4080 tp->t_end_info_status |= bit;
4081 break;
4082 }
4083 }
4084 }
4085
4086 int
tcp_can_enable_pacing(void)4087 tcp_can_enable_pacing(void)
4088 {
4089
4090 if ((tcp_pacing_limit == -1) ||
4091 (tcp_pacing_limit > number_of_tcp_connections_pacing)) {
4092 atomic_fetchadd_int(&number_of_tcp_connections_pacing, 1);
4093 shadow_num_connections = number_of_tcp_connections_pacing;
4094 return (1);
4095 } else {
4096 counter_u64_add(tcp_pacing_failures, 1);
4097 return (0);
4098 }
4099 }
4100
4101 static uint8_t tcp_pacing_warning = 0;
4102
4103 void
tcp_decrement_paced_conn(void)4104 tcp_decrement_paced_conn(void)
4105 {
4106 uint32_t ret;
4107
4108 ret = atomic_fetchadd_int(&number_of_tcp_connections_pacing, -1);
4109 shadow_num_connections = number_of_tcp_connections_pacing;
4110 KASSERT(ret != 0, ("tcp_paced_connection_exits -1 would cause wrap?"));
4111 if (ret == 0) {
4112 if (tcp_pacing_limit != -1) {
4113 printf("Warning all pacing is now disabled, count decrements invalidly!\n");
4114 tcp_pacing_limit = 0;
4115 } else if (tcp_pacing_warning == 0) {
4116 printf("Warning pacing count is invalid, invalid decrement\n");
4117 tcp_pacing_warning = 1;
4118 }
4119 }
4120 }
4121
4122 static void
tcp_default_switch_failed(struct tcpcb * tp)4123 tcp_default_switch_failed(struct tcpcb *tp)
4124 {
4125 /*
4126 * If a switch fails we only need to
4127 * care about two things:
4128 * a) The t_flags2
4129 * and
4130 * b) The timer granularity.
4131 * Timeouts, at least for now, don't use the
4132 * old callout system in the other stacks so
4133 * those are hopefully safe.
4134 */
4135 tcp_lro_features_off(tp);
4136 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
4137 }
4138
4139 #ifdef TCP_ACCOUNTING
4140 int
tcp_do_ack_accounting(struct tcpcb * tp,struct tcphdr * th,struct tcpopt * to,uint32_t tiwin,int mss)4141 tcp_do_ack_accounting(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to, uint32_t tiwin, int mss)
4142 {
4143 if (SEQ_LT(th->th_ack, tp->snd_una)) {
4144 /* Do we have a SACK? */
4145 if (to->to_flags & TOF_SACK) {
4146 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4147 tp->tcp_cnt_counters[ACK_SACK]++;
4148 }
4149 return (ACK_SACK);
4150 } else {
4151 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4152 tp->tcp_cnt_counters[ACK_BEHIND]++;
4153 }
4154 return (ACK_BEHIND);
4155 }
4156 } else if (th->th_ack == tp->snd_una) {
4157 /* Do we have a SACK? */
4158 if (to->to_flags & TOF_SACK) {
4159 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4160 tp->tcp_cnt_counters[ACK_SACK]++;
4161 }
4162 return (ACK_SACK);
4163 } else if (tiwin != tp->snd_wnd) {
4164 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4165 tp->tcp_cnt_counters[ACK_RWND]++;
4166 }
4167 return (ACK_RWND);
4168 } else {
4169 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4170 tp->tcp_cnt_counters[ACK_DUPACK]++;
4171 }
4172 return (ACK_DUPACK);
4173 }
4174 } else {
4175 if (!SEQ_GT(th->th_ack, tp->snd_max)) {
4176 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4177 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((th->th_ack - tp->snd_una) + mss - 1)/mss);
4178 }
4179 }
4180 if (to->to_flags & TOF_SACK) {
4181 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4182 tp->tcp_cnt_counters[ACK_CUMACK_SACK]++;
4183 }
4184 return (ACK_CUMACK_SACK);
4185 } else {
4186 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4187 tp->tcp_cnt_counters[ACK_CUMACK]++;
4188 }
4189 return (ACK_CUMACK);
4190 }
4191 }
4192 }
4193 #endif
4194
4195 void
tcp_change_time_units(struct tcpcb * tp,int granularity)4196 tcp_change_time_units(struct tcpcb *tp, int granularity)
4197 {
4198 if (tp->t_tmr_granularity == granularity) {
4199 /* We are there */
4200 return;
4201 }
4202 if (granularity == TCP_TMR_GRANULARITY_USEC) {
4203 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS),
4204 ("Granularity is not TICKS its %u in tp:%p",
4205 tp->t_tmr_granularity, tp));
4206 tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
4207 if (tp->t_srtt > 1) {
4208 uint32_t val, frac;
4209
4210 val = tp->t_srtt >> TCP_RTT_SHIFT;
4211 frac = tp->t_srtt & 0x1f;
4212 tp->t_srtt = TICKS_2_USEC(val);
4213 /*
4214 * frac is the fractional part of the srtt (if any)
4215 * but its in ticks and every bit represents
4216 * 1/32nd of a hz.
4217 */
4218 if (frac) {
4219 if (hz == 1000) {
4220 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4221 } else {
4222 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4223 }
4224 tp->t_srtt += frac;
4225 }
4226 }
4227 if (tp->t_rttvar) {
4228 uint32_t val, frac;
4229
4230 val = tp->t_rttvar >> TCP_RTTVAR_SHIFT;
4231 frac = tp->t_rttvar & 0x1f;
4232 tp->t_rttvar = TICKS_2_USEC(val);
4233 /*
4234 * frac is the fractional part of the srtt (if any)
4235 * but its in ticks and every bit represents
4236 * 1/32nd of a hz.
4237 */
4238 if (frac) {
4239 if (hz == 1000) {
4240 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4241 } else {
4242 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4243 }
4244 tp->t_rttvar += frac;
4245 }
4246 }
4247 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_USEC;
4248 } else if (granularity == TCP_TMR_GRANULARITY_TICKS) {
4249 /* Convert back to ticks, with */
4250 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_USEC),
4251 ("Granularity is not USEC its %u in tp:%p",
4252 tp->t_tmr_granularity, tp));
4253 if (tp->t_srtt > 1) {
4254 uint32_t val, frac;
4255
4256 val = USEC_2_TICKS(tp->t_srtt);
4257 frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4258 tp->t_srtt = val << TCP_RTT_SHIFT;
4259 /*
4260 * frac is the fractional part here is left
4261 * over from converting to hz and shifting.
4262 * We need to convert this to the 5 bit
4263 * remainder.
4264 */
4265 if (frac) {
4266 if (hz == 1000) {
4267 frac = (((uint64_t)frac * (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4268 } else {
4269 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4270 }
4271 tp->t_srtt += frac;
4272 }
4273 }
4274 if (tp->t_rttvar) {
4275 uint32_t val, frac;
4276
4277 val = USEC_2_TICKS(tp->t_rttvar);
4278 frac = tp->t_rttvar % (HPTS_USEC_IN_SEC / hz);
4279 tp->t_rttvar = val << TCP_RTTVAR_SHIFT;
4280 /*
4281 * frac is the fractional part here is left
4282 * over from converting to hz and shifting.
4283 * We need to convert this to the 4 bit
4284 * remainder.
4285 */
4286 if (frac) {
4287 if (hz == 1000) {
4288 frac = (((uint64_t)frac * (uint64_t)TCP_RTTVAR_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4289 } else {
4290 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTTVAR_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4291 }
4292 tp->t_rttvar += frac;
4293 }
4294 }
4295 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow);
4296 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
4297 }
4298 #ifdef INVARIANTS
4299 else {
4300 panic("Unknown granularity:%d tp:%p",
4301 granularity, tp);
4302 }
4303 #endif
4304 }
4305
4306 void
tcp_handle_orphaned_packets(struct tcpcb * tp)4307 tcp_handle_orphaned_packets(struct tcpcb *tp)
4308 {
4309 struct mbuf *save, *m, *prev;
4310 /*
4311 * Called when a stack switch is occuring from the fini()
4312 * of the old stack. We assue the init() as already been
4313 * run of the new stack and it has set the t_flags2 to
4314 * what it supports. This function will then deal with any
4315 * differences i.e. cleanup packets that maybe queued that
4316 * the newstack does not support.
4317 */
4318
4319 if (tp->t_flags2 & TF2_MBUF_L_ACKS)
4320 return;
4321 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) == 0 &&
4322 !STAILQ_EMPTY(&tp->t_inqueue)) {
4323 /*
4324 * It is unsafe to process the packets since a
4325 * reset may be lurking in them (its rare but it
4326 * can occur). If we were to find a RST, then we
4327 * would end up dropping the connection and the
4328 * INP lock, so when we return the caller (tcp_usrreq)
4329 * will blow up when it trys to unlock the inp.
4330 * This new stack does not do any fancy LRO features
4331 * so all we can do is toss the packets.
4332 */
4333 m = STAILQ_FIRST(&tp->t_inqueue);
4334 STAILQ_INIT(&tp->t_inqueue);
4335 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, save)
4336 m_freem(m);
4337 } else {
4338 /*
4339 * Here we have a stack that does mbuf queuing but
4340 * does not support compressed ack's. We must
4341 * walk all the mbufs and discard any compressed acks.
4342 */
4343 STAILQ_FOREACH_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) {
4344 if (m->m_flags & M_ACKCMP) {
4345 if (m == STAILQ_FIRST(&tp->t_inqueue))
4346 STAILQ_REMOVE_HEAD(&tp->t_inqueue,
4347 m_stailqpkt);
4348 else
4349 STAILQ_REMOVE_AFTER(&tp->t_inqueue,
4350 prev, m_stailqpkt);
4351 m_freem(m);
4352 } else
4353 prev = m;
4354 }
4355 }
4356 }
4357
4358 #ifdef TCP_REQUEST_TRK
4359 uint32_t
tcp_estimate_tls_overhead(struct socket * so,uint64_t tls_usr_bytes)4360 tcp_estimate_tls_overhead(struct socket *so, uint64_t tls_usr_bytes)
4361 {
4362 #ifdef KERN_TLS
4363 struct ktls_session *tls;
4364 uint32_t rec_oh, records;
4365
4366 tls = so->so_snd.sb_tls_info;
4367 if (tls == NULL)
4368 return (0);
4369
4370 rec_oh = tls->params.tls_hlen + tls->params.tls_tlen;
4371 records = ((tls_usr_bytes + tls->params.max_frame_len - 1)/tls->params.max_frame_len);
4372 return (records * rec_oh);
4373 #else
4374 return (0);
4375 #endif
4376 }
4377
4378 extern uint32_t tcp_stale_entry_time;
4379 uint32_t tcp_stale_entry_time = 250000;
4380 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, usrlog_stale, CTLFLAG_RW,
4381 &tcp_stale_entry_time, 250000, "Time that a tcpreq entry without a sendfile ages out");
4382
4383 void
tcp_req_log_req_info(struct tcpcb * tp,struct tcp_sendfile_track * req,uint16_t slot,uint8_t val,uint64_t offset,uint64_t nbytes)4384 tcp_req_log_req_info(struct tcpcb *tp, struct tcp_sendfile_track *req,
4385 uint16_t slot, uint8_t val, uint64_t offset, uint64_t nbytes)
4386 {
4387 if (tcp_bblogging_on(tp)) {
4388 union tcp_log_stackspecific log;
4389 struct timeval tv;
4390
4391 memset(&log, 0, sizeof(log));
4392 log.u_bbr.inhpts = tcp_in_hpts(tp);
4393 log.u_bbr.flex8 = val;
4394 log.u_bbr.rttProp = req->timestamp;
4395 log.u_bbr.delRate = req->start;
4396 log.u_bbr.cur_del_rate = req->end;
4397 log.u_bbr.flex1 = req->start_seq;
4398 log.u_bbr.flex2 = req->end_seq;
4399 log.u_bbr.flex3 = req->flags;
4400 log.u_bbr.flex4 = ((req->localtime >> 32) & 0x00000000ffffffff);
4401 log.u_bbr.flex5 = (req->localtime & 0x00000000ffffffff);
4402 log.u_bbr.flex7 = slot;
4403 log.u_bbr.bw_inuse = offset;
4404 /* nbytes = flex6 | epoch */
4405 log.u_bbr.flex6 = ((nbytes >> 32) & 0x00000000ffffffff);
4406 log.u_bbr.epoch = (nbytes & 0x00000000ffffffff);
4407 /* cspr = lt_epoch | pkts_out */
4408 log.u_bbr.lt_epoch = ((req->cspr >> 32) & 0x00000000ffffffff);
4409 log.u_bbr.pkts_out |= (req->cspr & 0x00000000ffffffff);
4410 log.u_bbr.applimited = tp->t_tcpreq_closed;
4411 log.u_bbr.applimited <<= 8;
4412 log.u_bbr.applimited |= tp->t_tcpreq_open;
4413 log.u_bbr.applimited <<= 8;
4414 log.u_bbr.applimited |= tp->t_tcpreq_req;
4415 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4416 TCP_LOG_EVENTP(tp, NULL,
4417 &tptosocket(tp)->so_rcv,
4418 &tptosocket(tp)->so_snd,
4419 TCP_LOG_REQ_T, 0,
4420 0, &log, false, &tv);
4421 }
4422 }
4423
4424 void
tcp_req_free_a_slot(struct tcpcb * tp,struct tcp_sendfile_track * ent)4425 tcp_req_free_a_slot(struct tcpcb *tp, struct tcp_sendfile_track *ent)
4426 {
4427 if (tp->t_tcpreq_req > 0)
4428 tp->t_tcpreq_req--;
4429 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4430 if (tp->t_tcpreq_open > 0)
4431 tp->t_tcpreq_open--;
4432 } else {
4433 if (tp->t_tcpreq_closed > 0)
4434 tp->t_tcpreq_closed--;
4435 }
4436 ent->flags = TCP_TRK_TRACK_FLG_EMPTY;
4437 }
4438
4439 static void
tcp_req_check_for_stale_entries(struct tcpcb * tp,uint64_t ts,int rm_oldest)4440 tcp_req_check_for_stale_entries(struct tcpcb *tp, uint64_t ts, int rm_oldest)
4441 {
4442 struct tcp_sendfile_track *ent;
4443 uint64_t time_delta, oldest_delta;
4444 int i, oldest, oldest_set = 0, cnt_rm = 0;
4445
4446 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4447 ent = &tp->t_tcpreq_info[i];
4448 if (ent->flags != TCP_TRK_TRACK_FLG_USED) {
4449 /*
4450 * We only care about closed end ranges
4451 * that are allocated and have no sendfile
4452 * ever touching them. They would be in
4453 * state USED.
4454 */
4455 continue;
4456 }
4457 if (ts >= ent->localtime)
4458 time_delta = ts - ent->localtime;
4459 else
4460 time_delta = 0;
4461 if (time_delta &&
4462 ((oldest_delta < time_delta) || (oldest_set == 0))) {
4463 oldest_set = 1;
4464 oldest = i;
4465 oldest_delta = time_delta;
4466 }
4467 if (tcp_stale_entry_time && (time_delta >= tcp_stale_entry_time)) {
4468 /*
4469 * No sendfile in a our time-limit
4470 * time to purge it.
4471 */
4472 cnt_rm++;
4473 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4474 time_delta, 0);
4475 tcp_req_free_a_slot(tp, ent);
4476 }
4477 }
4478 if ((cnt_rm == 0) && rm_oldest && oldest_set) {
4479 ent = &tp->t_tcpreq_info[oldest];
4480 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4481 oldest_delta, 1);
4482 tcp_req_free_a_slot(tp, ent);
4483 }
4484 }
4485
4486 int
tcp_req_check_for_comp(struct tcpcb * tp,tcp_seq ack_point)4487 tcp_req_check_for_comp(struct tcpcb *tp, tcp_seq ack_point)
4488 {
4489 int i, ret = 0;
4490 struct tcp_sendfile_track *ent;
4491
4492 /* Clean up any old closed end requests that are now completed */
4493 if (tp->t_tcpreq_req == 0)
4494 return (0);
4495 if (tp->t_tcpreq_closed == 0)
4496 return (0);
4497 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4498 ent = &tp->t_tcpreq_info[i];
4499 /* Skip empty ones */
4500 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4501 continue;
4502 /* Skip open ones */
4503 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN)
4504 continue;
4505 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4506 /* We are past it -- free it */
4507 tcp_req_log_req_info(tp, ent,
4508 i, TCP_TRK_REQ_LOG_FREED, 0, 0);
4509 tcp_req_free_a_slot(tp, ent);
4510 ret++;
4511 }
4512 }
4513 return (ret);
4514 }
4515
4516 int
tcp_req_is_entry_comp(struct tcpcb * tp,struct tcp_sendfile_track * ent,tcp_seq ack_point)4517 tcp_req_is_entry_comp(struct tcpcb *tp, struct tcp_sendfile_track *ent, tcp_seq ack_point)
4518 {
4519 if (tp->t_tcpreq_req == 0)
4520 return (-1);
4521 if (tp->t_tcpreq_closed == 0)
4522 return (-1);
4523 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4524 return (-1);
4525 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4526 return (1);
4527 }
4528 return (0);
4529 }
4530
4531 struct tcp_sendfile_track *
tcp_req_find_a_req_that_is_completed_by(struct tcpcb * tp,tcp_seq th_ack,int * ip)4532 tcp_req_find_a_req_that_is_completed_by(struct tcpcb *tp, tcp_seq th_ack, int *ip)
4533 {
4534 /*
4535 * Given an ack point (th_ack) walk through our entries and
4536 * return the first one found that th_ack goes past the
4537 * end_seq.
4538 */
4539 struct tcp_sendfile_track *ent;
4540 int i;
4541
4542 if (tp->t_tcpreq_req == 0) {
4543 /* none open */
4544 return (NULL);
4545 }
4546 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4547 ent = &tp->t_tcpreq_info[i];
4548 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4549 continue;
4550 if ((ent->flags & TCP_TRK_TRACK_FLG_OPEN) == 0) {
4551 if (SEQ_GEQ(th_ack, ent->end_seq)) {
4552 *ip = i;
4553 return (ent);
4554 }
4555 }
4556 }
4557 return (NULL);
4558 }
4559
4560 struct tcp_sendfile_track *
tcp_req_find_req_for_seq(struct tcpcb * tp,tcp_seq seq)4561 tcp_req_find_req_for_seq(struct tcpcb *tp, tcp_seq seq)
4562 {
4563 struct tcp_sendfile_track *ent;
4564 int i;
4565
4566 if (tp->t_tcpreq_req == 0) {
4567 /* none open */
4568 return (NULL);
4569 }
4570 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4571 ent = &tp->t_tcpreq_info[i];
4572 tcp_req_log_req_info(tp, ent, i, TCP_TRK_REQ_LOG_SEARCH,
4573 (uint64_t)seq, 0);
4574 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4575 continue;
4576 }
4577 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4578 /*
4579 * An open end request only needs to
4580 * match the beginning seq or be
4581 * all we have (once we keep going on
4582 * a open end request we may have a seq
4583 * wrap).
4584 */
4585 if ((SEQ_GEQ(seq, ent->start_seq)) ||
4586 (tp->t_tcpreq_closed == 0))
4587 return (ent);
4588 } else {
4589 /*
4590 * For this one we need to
4591 * be a bit more careful if its
4592 * completed at least.
4593 */
4594 if ((SEQ_GEQ(seq, ent->start_seq)) &&
4595 (SEQ_LT(seq, ent->end_seq))) {
4596 return (ent);
4597 }
4598 }
4599 }
4600 return (NULL);
4601 }
4602
4603 /* Should this be in its own file tcp_req.c ? */
4604 struct tcp_sendfile_track *
tcp_req_alloc_req_full(struct tcpcb * tp,struct tcp_snd_req * req,uint64_t ts,int rec_dups)4605 tcp_req_alloc_req_full(struct tcpcb *tp, struct tcp_snd_req *req, uint64_t ts, int rec_dups)
4606 {
4607 struct tcp_sendfile_track *fil;
4608 int i, allocated;
4609
4610 /* In case the stack does not check for completions do so now */
4611 tcp_req_check_for_comp(tp, tp->snd_una);
4612 /* Check for stale entries */
4613 if (tp->t_tcpreq_req)
4614 tcp_req_check_for_stale_entries(tp, ts,
4615 (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ));
4616 /* Check to see if this is a duplicate of one not started */
4617 if (tp->t_tcpreq_req) {
4618 for (i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4619 fil = &tp->t_tcpreq_info[i];
4620 if (fil->flags != TCP_TRK_TRACK_FLG_USED)
4621 continue;
4622 if ((fil->timestamp == req->timestamp) &&
4623 (fil->start == req->start) &&
4624 ((fil->flags & TCP_TRK_TRACK_FLG_OPEN) ||
4625 (fil->end == req->end))) {
4626 /*
4627 * We already have this request
4628 * and it has not been started with sendfile.
4629 * This probably means the user was returned
4630 * a 4xx of some sort and its going to age
4631 * out, lets not duplicate it.
4632 */
4633 return (fil);
4634 }
4635 }
4636 }
4637 /* Ok if there is no room at the inn we are in trouble */
4638 if (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ) {
4639 tcp_trace_point(tp, TCP_TP_REQ_LOG_FAIL);
4640 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4641 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i],
4642 i, TCP_TRK_REQ_LOG_ALLOCFAIL, 0, 0);
4643 }
4644 return (NULL);
4645 }
4646 for (i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4647 fil = &tp->t_tcpreq_info[i];
4648 if (fil->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4649 allocated = 1;
4650 fil->flags = TCP_TRK_TRACK_FLG_USED;
4651 fil->timestamp = req->timestamp;
4652 fil->localtime = ts;
4653 fil->start = req->start;
4654 if (req->flags & TCP_LOG_HTTPD_RANGE_END) {
4655 fil->end = req->end;
4656 } else {
4657 fil->end = 0;
4658 fil->flags |= TCP_TRK_TRACK_FLG_OPEN;
4659 }
4660 /*
4661 * We can set the min boundaries to the TCP Sequence space,
4662 * but it might be found to be further up when sendfile
4663 * actually runs on this range (if it ever does).
4664 */
4665 fil->sbcc_at_s = tptosocket(tp)->so_snd.sb_ccc;
4666 fil->start_seq = tp->snd_una +
4667 tptosocket(tp)->so_snd.sb_ccc;
4668 fil->end_seq = (fil->start_seq + ((uint32_t)(fil->end - fil->start)));
4669 if (tptosocket(tp)->so_snd.sb_tls_info) {
4670 /*
4671 * This session is doing TLS. Take a swag guess
4672 * at the overhead.
4673 */
4674 fil->end_seq += tcp_estimate_tls_overhead(
4675 tptosocket(tp), (fil->end - fil->start));
4676 }
4677 tp->t_tcpreq_req++;
4678 if (fil->flags & TCP_TRK_TRACK_FLG_OPEN)
4679 tp->t_tcpreq_open++;
4680 else
4681 tp->t_tcpreq_closed++;
4682 tcp_req_log_req_info(tp, fil, i,
4683 TCP_TRK_REQ_LOG_NEW, 0, 0);
4684 break;
4685 } else
4686 fil = NULL;
4687 }
4688 return (fil);
4689 }
4690
4691 void
tcp_req_alloc_req(struct tcpcb * tp,union tcp_log_userdata * user,uint64_t ts)4692 tcp_req_alloc_req(struct tcpcb *tp, union tcp_log_userdata *user, uint64_t ts)
4693 {
4694 (void)tcp_req_alloc_req_full(tp, &user->tcp_req, ts, 1);
4695 }
4696 #endif
4697
4698 void
tcp_log_socket_option(struct tcpcb * tp,uint32_t option_num,uint32_t option_val,int err)4699 tcp_log_socket_option(struct tcpcb *tp, uint32_t option_num, uint32_t option_val, int err)
4700 {
4701 if (tcp_bblogging_on(tp)) {
4702 struct tcp_log_buffer *l;
4703
4704 l = tcp_log_event(tp, NULL,
4705 &tptosocket(tp)->so_rcv,
4706 &tptosocket(tp)->so_snd,
4707 TCP_LOG_SOCKET_OPT,
4708 err, 0, NULL, 1,
4709 NULL, NULL, 0, NULL);
4710 if (l) {
4711 l->tlb_flex1 = option_num;
4712 l->tlb_flex2 = option_val;
4713 }
4714 }
4715 }
4716
4717 uint32_t
tcp_get_srtt(struct tcpcb * tp,int granularity)4718 tcp_get_srtt(struct tcpcb *tp, int granularity)
4719 {
4720 uint32_t srtt;
4721
4722 KASSERT(granularity == TCP_TMR_GRANULARITY_USEC ||
4723 granularity == TCP_TMR_GRANULARITY_TICKS,
4724 ("%s: called with unexpected granularity %d", __func__,
4725 granularity));
4726
4727 srtt = tp->t_srtt;
4728
4729 /*
4730 * We only support two granularities. If the stored granularity
4731 * does not match the granularity requested by the caller,
4732 * convert the stored value to the requested unit of granularity.
4733 */
4734 if (tp->t_tmr_granularity != granularity) {
4735 if (granularity == TCP_TMR_GRANULARITY_USEC)
4736 srtt = TICKS_2_USEC(srtt);
4737 else
4738 srtt = USEC_2_TICKS(srtt);
4739 }
4740
4741 /*
4742 * If the srtt is stored with ticks granularity, we need to
4743 * unshift to get the actual value. We do this after the
4744 * conversion above (if one was necessary) in order to maximize
4745 * precision.
4746 */
4747 if (tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS)
4748 srtt = srtt >> TCP_RTT_SHIFT;
4749
4750 return (srtt);
4751 }
4752
4753 void
tcp_account_for_send(struct tcpcb * tp,uint32_t len,uint8_t is_rxt,uint8_t is_tlp,bool hw_tls)4754 tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt,
4755 uint8_t is_tlp, bool hw_tls)
4756 {
4757
4758 if (is_tlp) {
4759 tp->t_sndtlppack++;
4760 tp->t_sndtlpbyte += len;
4761 }
4762 /* To get total bytes sent you must add t_snd_rxt_bytes to t_sndbytes */
4763 if (is_rxt)
4764 tp->t_snd_rxt_bytes += len;
4765 else
4766 tp->t_sndbytes += len;
4767
4768 #ifdef KERN_TLS
4769 if (hw_tls && is_rxt && len != 0) {
4770 uint64_t rexmit_percent;
4771
4772 rexmit_percent = (1000ULL * tp->t_snd_rxt_bytes) /
4773 (10ULL * (tp->t_snd_rxt_bytes + tp->t_sndbytes));
4774 if (rexmit_percent > ktls_ifnet_max_rexmit_pct)
4775 ktls_disable_ifnet(tp);
4776 }
4777 #endif
4778 }
4779