1 /*-
2 * Copyright (c) 2016-2018 Netflix, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * Author: Lawrence Stewart <lstewart@netflix.com>
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/arb.h>
34 #include <sys/errno.h>
35 #include <sys/malloc.h>
36 #include <sys/qmath.h>
37 #include <sys/queue.h>
38 #include <sys/socket.h>
39 #include <sys/socketvar.h>
40 #include <sys/sysctl.h>
41 #ifdef _KERNEL
42 #include <sys/kernel.h>
43 #include <sys/lock.h>
44 #include <sys/rmlock.h>
45 #include <sys/systm.h>
46 #endif
47 #include <sys/stats.h>
48
49 #include <net/vnet.h>
50
51 #include <netinet/in.h>
52 #include <netinet/in_pcb.h>
53 #include <netinet/tcp.h>
54 #include <netinet/tcp_var.h>
55
56 #include <netinet/cc/cc.h>
57
58 VNET_DEFINE(int, tcp_perconn_stats_dflt_tpl) = -1;
59
60 #ifndef _KERNEL
61 #define V_tcp_perconn_stats_enable VNET(tcp_perconn_stats_enable)
62 #define V_tcp_perconn_stats_dflt_tpl VNET(tcp_perconn_stats_dflt_tpl)
63 #else /* _KERNEL */
64
65 VNET_DEFINE(int, tcp_perconn_stats_enable) = 2;
66 VNET_DEFINE_STATIC(struct stats_tpl_sample_rate *, tcp_perconn_stats_sample_rates);
67 VNET_DEFINE_STATIC(int, tcp_stats_nrates) = 0;
68 #define V_tcp_perconn_stats_sample_rates VNET(tcp_perconn_stats_sample_rates)
69 #define V_tcp_stats_nrates VNET(tcp_stats_nrates)
70
71 static struct rmlock tcp_stats_tpl_sampling_lock;
72 static int tcp_stats_tpl_sr_cb(enum stats_tpl_sr_cb_action action,
73 struct stats_tpl_sample_rate **rates, int *nrates, void *ctx);
74
75 SYSCTL_INT(_net_inet_tcp, OID_AUTO, perconn_stats_enable,
76 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_perconn_stats_enable), 0,
77 "Enable per-connection TCP stats gathering; 1 enables for all connections, "
78 "2 enables random sampling across log id connection groups");
79 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, perconn_stats_sample_rates,
80 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, tcp_stats_tpl_sr_cb,
81 sizeof(struct rm_priotracker), stats_tpl_sample_rates, "A",
82 "TCP stats per template random sampling rates, in CSV tpl_spec=percent "
83 "key-value pairs (see stats(9) for template spec details)");
84 #endif /* _KERNEL */
85
86 #ifdef _KERNEL
87 int
88 #else
89 static int
90 /* Ensure all templates are also added to the userland template list. */
91 __attribute__ ((constructor))
92 #endif
tcp_stats_init()93 tcp_stats_init()
94 {
95 int err, lasterr;
96
97 err = lasterr = 0;
98
99 V_tcp_perconn_stats_dflt_tpl = stats_tpl_alloc("TCP_DEFAULT", 0);
100 if (V_tcp_perconn_stats_dflt_tpl < 0)
101 return (-V_tcp_perconn_stats_dflt_tpl);
102
103 struct voistatspec vss_sum[] = {
104 STATS_VSS_SUM(),
105 };
106 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
107 VOI_TCP_TXPB, "TCP_TXPB", VSD_DTYPE_INT_U64,
108 NVSS(vss_sum), vss_sum, 0);
109 lasterr = err ? err : lasterr;
110 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
111 VOI_TCP_RETXPB, "TCP_RETXPB", VSD_DTYPE_INT_U32,
112 NVSS(vss_sum), vss_sum, 0);
113 lasterr = err ? err : lasterr;
114
115 struct voistatspec vss_max[] = {
116 STATS_VSS_MAX(),
117 };
118 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
119 VOI_TCP_FRWIN, "TCP_FRWIN", VSD_DTYPE_INT_ULONG,
120 NVSS(vss_max), vss_max, 0);
121 lasterr = err ? err : lasterr;
122 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
123 VOI_TCP_LCWIN, "TCP_LCWIN", VSD_DTYPE_INT_ULONG,
124 NVSS(vss_max), vss_max, 0);
125 lasterr = err ? err : lasterr;
126
127 struct voistatspec vss_rtt[] = {
128 STATS_VSS_MAX(),
129 STATS_VSS_MIN(),
130 STATS_VSS_TDGSTCLUST32(20, 4),
131 };
132 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
133 VOI_TCP_RTT, "TCP_RTT", VSD_DTYPE_INT_U32,
134 NVSS(vss_rtt), vss_rtt, 0);
135 lasterr = err ? err : lasterr;
136
137 struct voistatspec vss_congsig[] = {
138 STATS_VSS_DVHIST32_USR(HBKTS(DVBKT(CC_ECN), DVBKT(CC_RTO),
139 DVBKT(CC_RTO_ERR), DVBKT(CC_NDUPACK)), 0)
140 };
141 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
142 VOI_TCP_CSIG, "TCP_CSIG", VSD_DTYPE_INT_U32,
143 NVSS(vss_congsig), vss_congsig, 0);
144 lasterr = err ? err : lasterr;
145
146 struct voistatspec vss_gput[] = {
147 STATS_VSS_MAX(),
148 STATS_VSS_TDGSTCLUST32(20, 4),
149 };
150 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
151 VOI_TCP_GPUT, "TCP_GPUT", VSD_DTYPE_INT_U32,
152 NVSS(vss_gput), vss_gput, 0);
153 lasterr = err ? err : lasterr;
154
155 struct voistatspec vss_gput_nd[] = {
156 STATS_VSS_TDGSTCLUST32(10, 4),
157 };
158 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
159 VOI_TCP_GPUT_ND, "TCP_GPUT_ND", VSD_DTYPE_INT_S32,
160 NVSS(vss_gput_nd), vss_gput_nd, 0);
161 lasterr = err ? err : lasterr;
162
163 struct voistatspec vss_windiff[] = {
164 STATS_VSS_CRHIST32_USR(HBKTS(CRBKT(0)), VSD_HIST_LBOUND_INF)
165 };
166 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
167 VOI_TCP_CALCFRWINDIFF, "TCP_CALCFRWINDIFF", VSD_DTYPE_INT_S32,
168 NVSS(vss_windiff), vss_windiff, 0);
169 lasterr = err ? err : lasterr;
170
171 struct voistatspec vss_acklen[] = {
172 STATS_VSS_MAX(),
173 STATS_VSS_CRHIST32_LIN(0, 9, 1, VSD_HIST_UBOUND_INF)
174 };
175 err |= stats_tpl_add_voistats(V_tcp_perconn_stats_dflt_tpl,
176 VOI_TCP_ACKLEN, "TCP_ACKLEN", VSD_DTYPE_INT_U32,
177 NVSS(vss_acklen), vss_acklen, 0);
178 lasterr = err ? err : lasterr;
179
180 return (lasterr);
181 }
182
183 #ifdef _KERNEL
184 int
tcp_stats_sample_rollthedice(struct tcpcb * tp,void * seed_bytes,size_t seed_len)185 tcp_stats_sample_rollthedice(struct tcpcb *tp, void *seed_bytes,
186 size_t seed_len)
187 {
188 struct rm_priotracker tracker;
189 int tpl;
190
191 tpl = -1;
192
193 if (V_tcp_stats_nrates > 0) {
194 rm_rlock(&tcp_stats_tpl_sampling_lock, &tracker);
195 tpl = stats_tpl_sample_rollthedice(V_tcp_perconn_stats_sample_rates,
196 V_tcp_stats_nrates, seed_bytes, seed_len);
197 rm_runlock(&tcp_stats_tpl_sampling_lock, &tracker);
198
199 if (tpl >= 0) {
200 INP_WLOCK_ASSERT(tp->t_inpcb);
201 if (tp->t_stats != NULL)
202 stats_blob_destroy(tp->t_stats);
203 tp->t_stats = stats_blob_alloc(tpl, 0);
204 if (tp->t_stats == NULL)
205 tpl = -ENOMEM;
206 }
207 }
208
209 return (tpl);
210 }
211
212 /*
213 * Callback function for stats_tpl_sample_rates() to interact with the TCP
214 * subsystem's stats template sample rates list.
215 */
216 int
tcp_stats_tpl_sr_cb(enum stats_tpl_sr_cb_action action,struct stats_tpl_sample_rate ** rates,int * nrates,void * ctx)217 tcp_stats_tpl_sr_cb(enum stats_tpl_sr_cb_action action,
218 struct stats_tpl_sample_rate **rates, int *nrates, void *ctx)
219 {
220 struct stats_tpl_sample_rate *old_rates;
221 int old_nrates;
222
223 if (ctx == NULL)
224 return (ENOMEM);
225
226 switch (action) {
227 case TPL_SR_RLOCKED_GET:
228 /*
229 * Return with rlock held i.e. this call must be paired with a
230 * "action == TPL_SR_RUNLOCK" call.
231 */
232 rm_assert(&tcp_stats_tpl_sampling_lock, RA_UNLOCKED);
233 rm_rlock(&tcp_stats_tpl_sampling_lock,
234 (struct rm_priotracker *)ctx);
235 /* FALLTHROUGH */
236 case TPL_SR_UNLOCKED_GET:
237 if (rates != NULL)
238 *rates = V_tcp_perconn_stats_sample_rates;
239 if (nrates != NULL)
240 *nrates = V_tcp_stats_nrates;
241 break;
242 case TPL_SR_RUNLOCK:
243 rm_assert(&tcp_stats_tpl_sampling_lock, RA_RLOCKED);
244 rm_runlock(&tcp_stats_tpl_sampling_lock,
245 (struct rm_priotracker *)ctx);
246 break;
247 case TPL_SR_PUT:
248 KASSERT(rates != NULL && nrates != NULL,
249 ("%s: PUT without new rates", __func__));
250 rm_assert(&tcp_stats_tpl_sampling_lock, RA_UNLOCKED);
251 if (rates == NULL || nrates == NULL)
252 return (EINVAL);
253 rm_wlock(&tcp_stats_tpl_sampling_lock);
254 old_rates = V_tcp_perconn_stats_sample_rates;
255 old_nrates = V_tcp_stats_nrates;
256 V_tcp_perconn_stats_sample_rates = *rates;
257 V_tcp_stats_nrates = *nrates;
258 rm_wunlock(&tcp_stats_tpl_sampling_lock);
259 *rates = old_rates;
260 *nrates = old_nrates;
261 break;
262 default:
263 return (EINVAL);
264 break;
265 }
266
267 return (0);
268 }
269
270 RM_SYSINIT(tcp_stats_tpl_sampling_lock, &tcp_stats_tpl_sampling_lock,
271 "tcp_stats_tpl_sampling_lock");
272 #endif /* _KERNEL */
273