1 /*-
2 * Copyright (c) 2015 Gleb Smirnoff <glebius@FreeBSD.org>
3 * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>
4 * Copyright (c) 1982, 1986, 1988, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * @(#)ip_input.c 8.2 (Berkeley) 1/4/94
32 */
33
34 #include <sys/cdefs.h>
35 #include "opt_rss.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/eventhandler.h>
40 #include <sys/kernel.h>
41 #include <sys/hash.h>
42 #include <sys/mbuf.h>
43 #include <sys/malloc.h>
44 #include <sys/limits.h>
45 #include <sys/lock.h>
46 #include <sys/mutex.h>
47 #include <sys/sysctl.h>
48 #include <sys/socket.h>
49
50 #include <net/if.h>
51 #include <net/if_var.h>
52 #include <net/rss_config.h>
53 #include <net/netisr.h>
54 #include <net/vnet.h>
55
56 #include <netinet/in.h>
57 #include <netinet/ip.h>
58 #include <netinet/ip_var.h>
59 #include <netinet/in_rss.h>
60 #ifdef MAC
61 #include <security/mac/mac_framework.h>
62 #endif
63
64 SYSCTL_DECL(_net_inet_ip);
65
66 /*
67 * Reassembly headers are stored in hash buckets.
68 */
69 #define IPREASS_NHASH_LOG2 10
70 #define IPREASS_NHASH (1 << IPREASS_NHASH_LOG2)
71 #define IPREASS_HMASK (IPREASS_NHASH - 1)
72
73 struct ipqbucket {
74 TAILQ_HEAD(ipqhead, ipq) head;
75 struct mtx lock;
76 int count;
77 };
78
79 VNET_DEFINE_STATIC(struct ipqbucket, ipq[IPREASS_NHASH]);
80 #define V_ipq VNET(ipq)
81 VNET_DEFINE_STATIC(uint32_t, ipq_hashseed);
82 #define V_ipq_hashseed VNET(ipq_hashseed)
83
84 #define IPQ_LOCK(i) mtx_lock(&V_ipq[i].lock)
85 #define IPQ_TRYLOCK(i) mtx_trylock(&V_ipq[i].lock)
86 #define IPQ_UNLOCK(i) mtx_unlock(&V_ipq[i].lock)
87 #define IPQ_LOCK_ASSERT(i) mtx_assert(&V_ipq[i].lock, MA_OWNED)
88
89 VNET_DEFINE_STATIC(int, ipreass_maxbucketsize);
90 #define V_ipreass_maxbucketsize VNET(ipreass_maxbucketsize)
91
92 void ipreass_init(void);
93 void ipreass_drain(void);
94 void ipreass_slowtimo(void);
95 #ifdef VIMAGE
96 void ipreass_destroy(void);
97 #endif
98 static int sysctl_maxfragpackets(SYSCTL_HANDLER_ARGS);
99 static int sysctl_maxfragbucketsize(SYSCTL_HANDLER_ARGS);
100 static void ipreass_zone_change(void *);
101 static void ipreass_drain_tomax(void);
102 static void ipq_free(struct ipqbucket *, struct ipq *);
103 static struct ipq * ipq_reuse(int);
104
105 static inline void
ipq_timeout(struct ipqbucket * bucket,struct ipq * fp)106 ipq_timeout(struct ipqbucket *bucket, struct ipq *fp)
107 {
108
109 IPSTAT_ADD(ips_fragtimeout, fp->ipq_nfrags);
110 ipq_free(bucket, fp);
111 }
112
113 static inline void
ipq_drop(struct ipqbucket * bucket,struct ipq * fp)114 ipq_drop(struct ipqbucket *bucket, struct ipq *fp)
115 {
116
117 IPSTAT_ADD(ips_fragdropped, fp->ipq_nfrags);
118 ipq_free(bucket, fp);
119 }
120
121 /*
122 * By default, limit the number of IP fragments across all reassembly
123 * queues to 1/32 of the total number of mbuf clusters.
124 *
125 * Limit the total number of reassembly queues per VNET to the
126 * IP fragment limit, but ensure the limit will not allow any bucket
127 * to grow above 100 items. (The bucket limit is
128 * IP_MAXFRAGPACKETS / (IPREASS_NHASH / 2), so the 50 is the correct
129 * multiplier to reach a 100-item limit.)
130 * The 100-item limit was chosen as brief testing seems to show that
131 * this produces "reasonable" performance on some subset of systems
132 * under DoS attack.
133 */
134 #define IP_MAXFRAGS (nmbclusters / 32)
135 #define IP_MAXFRAGPACKETS (imin(IP_MAXFRAGS, IPREASS_NHASH * 50))
136
137 static int maxfrags;
138 static u_int __exclusive_cache_line nfrags;
139 SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfrags, CTLFLAG_RW,
140 &maxfrags, 0,
141 "Maximum number of IPv4 fragments allowed across all reassembly queues");
142 SYSCTL_UINT(_net_inet_ip, OID_AUTO, curfrags, CTLFLAG_RD,
143 &nfrags, 0,
144 "Current number of IPv4 fragments across all reassembly queues");
145
146 VNET_DEFINE_STATIC(uma_zone_t, ipq_zone);
147 #define V_ipq_zone VNET(ipq_zone)
148 SYSCTL_PROC(_net_inet_ip, OID_AUTO, maxfragpackets,
149 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
150 NULL, 0, sysctl_maxfragpackets, "I",
151 "Maximum number of IPv4 fragment reassembly queue entries");
152 SYSCTL_UMA_CUR(_net_inet_ip, OID_AUTO, fragpackets, CTLFLAG_VNET,
153 &VNET_NAME(ipq_zone),
154 "Current number of IPv4 fragment reassembly queue entries");
155
156 VNET_DEFINE_STATIC(int, noreass);
157 #define V_noreass VNET(noreass)
158
159 VNET_DEFINE_STATIC(int, maxfragsperpacket);
160 #define V_maxfragsperpacket VNET(maxfragsperpacket)
161 SYSCTL_INT(_net_inet_ip, OID_AUTO, maxfragsperpacket, CTLFLAG_VNET | CTLFLAG_RW,
162 &VNET_NAME(maxfragsperpacket), 0,
163 "Maximum number of IPv4 fragments allowed per packet");
164 SYSCTL_PROC(_net_inet_ip, OID_AUTO, maxfragbucketsize,
165 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0,
166 sysctl_maxfragbucketsize, "I",
167 "Maximum number of IPv4 fragment reassembly queue entries per bucket");
168
169 /*
170 * Take incoming datagram fragment and try to reassemble it into
171 * whole datagram. If the argument is the first fragment or one
172 * in between the function will return NULL and store the mbuf
173 * in the fragment chain. If the argument is the last fragment
174 * the packet will be reassembled and the pointer to the new
175 * mbuf returned for further processing. Only m_tags attached
176 * to the first packet/fragment are preserved.
177 * The IP header is *NOT* adjusted out of iplen.
178 */
179 #define M_IP_FRAG M_PROTO9
180 struct mbuf *
ip_reass(struct mbuf * m)181 ip_reass(struct mbuf *m)
182 {
183 struct ip *ip;
184 struct mbuf *p, *q, *nq, *t;
185 struct ipq *fp;
186 struct ifnet *srcifp;
187 struct ipqhead *head;
188 int i, hlen, next, tmpmax;
189 u_int8_t ecn, ecn0;
190 uint32_t hash, hashkey[3];
191 #ifdef RSS
192 uint32_t rss_hash, rss_type;
193 #endif
194
195 /*
196 * If no reassembling or maxfragsperpacket are 0,
197 * never accept fragments.
198 * Also, drop packet if it would exceed the maximum
199 * number of fragments.
200 */
201 tmpmax = maxfrags;
202 if (V_noreass == 1 || V_maxfragsperpacket == 0 ||
203 (tmpmax >= 0 && atomic_load_int(&nfrags) >= (u_int)tmpmax)) {
204 IPSTAT_INC(ips_fragments);
205 IPSTAT_INC(ips_fragdropped);
206 m_freem(m);
207 return (NULL);
208 }
209
210 ip = mtod(m, struct ip *);
211 hlen = ip->ip_hl << 2;
212
213 /*
214 * Adjust ip_len to not reflect header,
215 * convert offset of this to bytes.
216 */
217 ip->ip_len = htons(ntohs(ip->ip_len) - hlen);
218 /*
219 * Make sure that fragments have a data length
220 * that's a non-zero multiple of 8 bytes, unless
221 * this is the last fragment.
222 */
223 if (ip->ip_len == htons(0) ||
224 ((ip->ip_off & htons(IP_MF)) && (ntohs(ip->ip_len) & 0x7) != 0)) {
225 IPSTAT_INC(ips_toosmall); /* XXX */
226 IPSTAT_INC(ips_fragdropped);
227 m_freem(m);
228 return (NULL);
229 }
230 if (ip->ip_off & htons(IP_MF))
231 m->m_flags |= M_IP_FRAG;
232 else
233 m->m_flags &= ~M_IP_FRAG;
234 ip->ip_off = htons(ntohs(ip->ip_off) << 3);
235
236 /*
237 * Make sure the fragment lies within a packet of valid size.
238 */
239 if (ntohs(ip->ip_len) + ntohs(ip->ip_off) > IP_MAXPACKET) {
240 IPSTAT_INC(ips_toolong);
241 IPSTAT_INC(ips_fragdropped);
242 m_freem(m);
243 return (NULL);
244 }
245
246 /*
247 * Store receive network interface pointer for later.
248 */
249 srcifp = m->m_pkthdr.rcvif;
250
251 /*
252 * Attempt reassembly; if it succeeds, proceed.
253 * ip_reass() will return a different mbuf.
254 */
255 IPSTAT_INC(ips_fragments);
256 m->m_pkthdr.PH_loc.ptr = ip;
257
258 /*
259 * Presence of header sizes in mbufs
260 * would confuse code below.
261 */
262 m->m_data += hlen;
263 m->m_len -= hlen;
264
265 hashkey[0] = ip->ip_src.s_addr;
266 hashkey[1] = ip->ip_dst.s_addr;
267 hashkey[2] = (uint32_t)ip->ip_p << 16;
268 hashkey[2] += ip->ip_id;
269 hash = jenkins_hash32(hashkey, nitems(hashkey), V_ipq_hashseed);
270 hash &= IPREASS_HMASK;
271 head = &V_ipq[hash].head;
272 IPQ_LOCK(hash);
273
274 /*
275 * Look for queue of fragments
276 * of this datagram.
277 */
278 TAILQ_FOREACH(fp, head, ipq_list)
279 if (ip->ip_id == fp->ipq_id &&
280 ip->ip_src.s_addr == fp->ipq_src.s_addr &&
281 ip->ip_dst.s_addr == fp->ipq_dst.s_addr &&
282 #ifdef MAC
283 mac_ipq_match(m, fp) &&
284 #endif
285 ip->ip_p == fp->ipq_p)
286 break;
287 /*
288 * If first fragment to arrive, create a reassembly queue.
289 */
290 if (fp == NULL) {
291 if (V_ipq[hash].count < V_ipreass_maxbucketsize)
292 fp = uma_zalloc(V_ipq_zone, M_NOWAIT);
293 if (fp == NULL)
294 fp = ipq_reuse(hash);
295 if (fp == NULL)
296 goto dropfrag;
297 #ifdef MAC
298 if (mac_ipq_init(fp, M_NOWAIT) != 0) {
299 uma_zfree(V_ipq_zone, fp);
300 fp = NULL;
301 goto dropfrag;
302 }
303 mac_ipq_create(m, fp);
304 #endif
305 TAILQ_INSERT_HEAD(head, fp, ipq_list);
306 V_ipq[hash].count++;
307 fp->ipq_nfrags = 1;
308 atomic_add_int(&nfrags, 1);
309 fp->ipq_ttl = IPFRAGTTL;
310 fp->ipq_p = ip->ip_p;
311 fp->ipq_id = ip->ip_id;
312 fp->ipq_src = ip->ip_src;
313 fp->ipq_dst = ip->ip_dst;
314 fp->ipq_frags = m;
315 if (m->m_flags & M_IP_FRAG)
316 fp->ipq_maxoff = -1;
317 else
318 fp->ipq_maxoff = ntohs(ip->ip_off) + ntohs(ip->ip_len);
319 m->m_nextpkt = NULL;
320 goto done;
321 } else {
322 /*
323 * If we already saw the last fragment, make sure
324 * this fragment's offset looks sane. Otherwise, if
325 * this is the last fragment, record its endpoint.
326 */
327 if (fp->ipq_maxoff > 0) {
328 i = ntohs(ip->ip_off) + ntohs(ip->ip_len);
329 if (((m->m_flags & M_IP_FRAG) && i >= fp->ipq_maxoff) ||
330 ((m->m_flags & M_IP_FRAG) == 0 &&
331 i != fp->ipq_maxoff)) {
332 fp = NULL;
333 goto dropfrag;
334 }
335 } else if ((m->m_flags & M_IP_FRAG) == 0)
336 fp->ipq_maxoff = ntohs(ip->ip_off) + ntohs(ip->ip_len);
337 fp->ipq_nfrags++;
338 atomic_add_int(&nfrags, 1);
339 #ifdef MAC
340 mac_ipq_update(m, fp);
341 #endif
342 }
343
344 #define GETIP(m) ((struct ip*)((m)->m_pkthdr.PH_loc.ptr))
345
346 /*
347 * Handle ECN by comparing this segment with the first one;
348 * if CE is set, do not lose CE.
349 * drop if CE and not-ECT are mixed for the same packet.
350 */
351 ecn = ip->ip_tos & IPTOS_ECN_MASK;
352 ecn0 = GETIP(fp->ipq_frags)->ip_tos & IPTOS_ECN_MASK;
353 if (ecn == IPTOS_ECN_CE) {
354 if (ecn0 == IPTOS_ECN_NOTECT)
355 goto dropfrag;
356 if (ecn0 != IPTOS_ECN_CE)
357 GETIP(fp->ipq_frags)->ip_tos |= IPTOS_ECN_CE;
358 }
359 if (ecn == IPTOS_ECN_NOTECT && ecn0 != IPTOS_ECN_NOTECT)
360 goto dropfrag;
361
362 /*
363 * Find a segment which begins after this one does.
364 */
365 for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt)
366 if (ntohs(GETIP(q)->ip_off) > ntohs(ip->ip_off))
367 break;
368
369 /*
370 * If there is a preceding segment, it may provide some of
371 * our data already. If so, drop the data from the incoming
372 * segment. If it provides all of our data, drop us, otherwise
373 * stick new segment in the proper place.
374 *
375 * If some of the data is dropped from the preceding
376 * segment, then it's checksum is invalidated.
377 */
378 if (p) {
379 i = ntohs(GETIP(p)->ip_off) + ntohs(GETIP(p)->ip_len) -
380 ntohs(ip->ip_off);
381 if (i > 0) {
382 if (i >= ntohs(ip->ip_len))
383 goto dropfrag;
384 m_adj(m, i);
385 m->m_pkthdr.csum_flags = 0;
386 ip->ip_off = htons(ntohs(ip->ip_off) + i);
387 ip->ip_len = htons(ntohs(ip->ip_len) - i);
388 }
389 m->m_nextpkt = p->m_nextpkt;
390 p->m_nextpkt = m;
391 } else {
392 m->m_nextpkt = fp->ipq_frags;
393 fp->ipq_frags = m;
394 }
395
396 /*
397 * While we overlap succeeding segments trim them or,
398 * if they are completely covered, dequeue them.
399 */
400 for (; q != NULL && ntohs(ip->ip_off) + ntohs(ip->ip_len) >
401 ntohs(GETIP(q)->ip_off); q = nq) {
402 i = (ntohs(ip->ip_off) + ntohs(ip->ip_len)) -
403 ntohs(GETIP(q)->ip_off);
404 if (i < ntohs(GETIP(q)->ip_len)) {
405 GETIP(q)->ip_len = htons(ntohs(GETIP(q)->ip_len) - i);
406 GETIP(q)->ip_off = htons(ntohs(GETIP(q)->ip_off) + i);
407 m_adj(q, i);
408 q->m_pkthdr.csum_flags = 0;
409 break;
410 }
411 nq = q->m_nextpkt;
412 m->m_nextpkt = nq;
413 IPSTAT_INC(ips_fragdropped);
414 fp->ipq_nfrags--;
415 atomic_subtract_int(&nfrags, 1);
416 m_freem(q);
417 }
418
419 /*
420 * Check for complete reassembly and perform frag per packet
421 * limiting.
422 *
423 * Frag limiting is performed here so that the nth frag has
424 * a chance to complete the packet before we drop the packet.
425 * As a result, n+1 frags are actually allowed per packet, but
426 * only n will ever be stored. (n = maxfragsperpacket.)
427 *
428 */
429 next = 0;
430 for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
431 if (ntohs(GETIP(q)->ip_off) != next) {
432 if (fp->ipq_nfrags > V_maxfragsperpacket)
433 ipq_drop(&V_ipq[hash], fp);
434 goto done;
435 }
436 next += ntohs(GETIP(q)->ip_len);
437 }
438 /* Make sure the last packet didn't have the IP_MF flag */
439 if (p->m_flags & M_IP_FRAG) {
440 if (fp->ipq_nfrags > V_maxfragsperpacket)
441 ipq_drop(&V_ipq[hash], fp);
442 goto done;
443 }
444
445 /*
446 * Reassembly is complete. Make sure the packet is a sane size.
447 */
448 q = fp->ipq_frags;
449 ip = GETIP(q);
450 if (next + (ip->ip_hl << 2) > IP_MAXPACKET) {
451 IPSTAT_INC(ips_toolong);
452 ipq_drop(&V_ipq[hash], fp);
453 goto done;
454 }
455
456 /*
457 * Concatenate fragments.
458 */
459 m = q;
460 t = m->m_next;
461 m->m_next = NULL;
462 m_cat(m, t);
463 nq = q->m_nextpkt;
464 q->m_nextpkt = NULL;
465 for (q = nq; q != NULL; q = nq) {
466 nq = q->m_nextpkt;
467 q->m_nextpkt = NULL;
468 m->m_pkthdr.csum_flags &= q->m_pkthdr.csum_flags;
469 m->m_pkthdr.csum_data += q->m_pkthdr.csum_data;
470 m_demote_pkthdr(q);
471 m_cat(m, q);
472 }
473 /*
474 * In order to do checksumming faster we do 'end-around carry' here
475 * (and not in for{} loop), though it implies we are not going to
476 * reassemble more than 64k fragments.
477 */
478 while (m->m_pkthdr.csum_data & 0xffff0000)
479 m->m_pkthdr.csum_data = (m->m_pkthdr.csum_data & 0xffff) +
480 (m->m_pkthdr.csum_data >> 16);
481 atomic_subtract_int(&nfrags, fp->ipq_nfrags);
482 #ifdef MAC
483 mac_ipq_reassemble(fp, m);
484 mac_ipq_destroy(fp);
485 #endif
486
487 /*
488 * Create header for new ip packet by modifying header of first
489 * packet; dequeue and discard fragment reassembly header.
490 * Make header visible.
491 */
492 ip->ip_len = htons((ip->ip_hl << 2) + next);
493 ip->ip_src = fp->ipq_src;
494 ip->ip_dst = fp->ipq_dst;
495 TAILQ_REMOVE(head, fp, ipq_list);
496 V_ipq[hash].count--;
497 uma_zfree(V_ipq_zone, fp);
498 m->m_len += (ip->ip_hl << 2);
499 m->m_data -= (ip->ip_hl << 2);
500 /* some debugging cruft by sklower, below, will go away soon */
501 if (m->m_flags & M_PKTHDR) { /* XXX this should be done elsewhere */
502 m_fixhdr(m);
503 /* set valid receive interface pointer */
504 m->m_pkthdr.rcvif = srcifp;
505 }
506 IPSTAT_INC(ips_reassembled);
507 IPQ_UNLOCK(hash);
508
509 #ifdef RSS
510 /*
511 * Query the RSS layer for the flowid / flowtype for the
512 * mbuf payload.
513 *
514 * For now, just assume we have to calculate a new one.
515 * Later on we should check to see if the assigned flowid matches
516 * what RSS wants for the given IP protocol and if so, just keep it.
517 *
518 * We then queue into the relevant netisr so it can be dispatched
519 * to the correct CPU.
520 *
521 * Note - this may return 1, which means the flowid in the mbuf
522 * is correct for the configured RSS hash types and can be used.
523 */
524 if (rss_mbuf_software_hash_v4(m, 0, &rss_hash, &rss_type) == 0) {
525 m->m_pkthdr.flowid = rss_hash;
526 M_HASHTYPE_SET(m, rss_type);
527 }
528
529 /*
530 * Queue/dispatch for reprocessing.
531 *
532 * Note: this is much slower than just handling the frame in the
533 * current receive context. It's likely worth investigating
534 * why this is.
535 */
536 netisr_dispatch(NETISR_IP_DIRECT, m);
537 return (NULL);
538 #endif
539
540 /* Handle in-line */
541 return (m);
542
543 dropfrag:
544 IPSTAT_INC(ips_fragdropped);
545 if (fp != NULL) {
546 fp->ipq_nfrags--;
547 atomic_subtract_int(&nfrags, 1);
548 }
549 m_freem(m);
550 done:
551 IPQ_UNLOCK(hash);
552 return (NULL);
553
554 #undef GETIP
555 }
556
557 /*
558 * Initialize IP reassembly structures.
559 */
560 void
ipreass_init(void)561 ipreass_init(void)
562 {
563 int max;
564
565 for (int i = 0; i < IPREASS_NHASH; i++) {
566 TAILQ_INIT(&V_ipq[i].head);
567 mtx_init(&V_ipq[i].lock, "IP reassembly", NULL,
568 MTX_DEF | MTX_DUPOK);
569 V_ipq[i].count = 0;
570 }
571 V_ipq_hashseed = arc4random();
572 V_maxfragsperpacket = 16;
573 V_ipq_zone = uma_zcreate("ipq", sizeof(struct ipq), NULL, NULL, NULL,
574 NULL, UMA_ALIGN_PTR, 0);
575 max = IP_MAXFRAGPACKETS;
576 max = uma_zone_set_max(V_ipq_zone, max);
577 V_ipreass_maxbucketsize = imax(max / (IPREASS_NHASH / 2), 1);
578
579 if (IS_DEFAULT_VNET(curvnet)) {
580 maxfrags = IP_MAXFRAGS;
581 EVENTHANDLER_REGISTER(nmbclusters_change, ipreass_zone_change,
582 NULL, EVENTHANDLER_PRI_ANY);
583 }
584 }
585
586 /*
587 * If a timer expires on a reassembly queue, discard it.
588 */
589 void
ipreass_slowtimo(void)590 ipreass_slowtimo(void)
591 {
592 struct ipq *fp, *tmp;
593
594 if (atomic_load_int(&nfrags) == 0)
595 return;
596
597 for (int i = 0; i < IPREASS_NHASH; i++) {
598 if (TAILQ_EMPTY(&V_ipq[i].head))
599 continue;
600 IPQ_LOCK(i);
601 TAILQ_FOREACH_SAFE(fp, &V_ipq[i].head, ipq_list, tmp)
602 if (--fp->ipq_ttl == 0)
603 ipq_timeout(&V_ipq[i], fp);
604 IPQ_UNLOCK(i);
605 }
606 }
607
608 /*
609 * Drain off all datagram fragments.
610 */
611 void
ipreass_drain(void)612 ipreass_drain(void)
613 {
614
615 for (int i = 0; i < IPREASS_NHASH; i++) {
616 IPQ_LOCK(i);
617 while(!TAILQ_EMPTY(&V_ipq[i].head))
618 ipq_drop(&V_ipq[i], TAILQ_FIRST(&V_ipq[i].head));
619 KASSERT(V_ipq[i].count == 0,
620 ("%s: V_ipq[%d] count %d (V_ipq=%p)", __func__, i,
621 V_ipq[i].count, V_ipq));
622 IPQ_UNLOCK(i);
623 }
624 }
625
626 /*
627 * Drain off all datagram fragments belonging to
628 * the given network interface.
629 */
630 static void
ipreass_cleanup(void * arg __unused,struct ifnet * ifp)631 ipreass_cleanup(void *arg __unused, struct ifnet *ifp)
632 {
633 struct ipq *fp, *temp;
634 struct mbuf *m;
635 int i;
636
637 KASSERT(ifp != NULL, ("%s: ifp is NULL", __func__));
638
639 CURVNET_SET_QUIET(ifp->if_vnet);
640
641 /*
642 * Skip processing if IPv4 reassembly is not initialised or
643 * torn down by ipreass_destroy().
644 */
645 if (V_ipq_zone == NULL) {
646 CURVNET_RESTORE();
647 return;
648 }
649
650 for (i = 0; i < IPREASS_NHASH; i++) {
651 IPQ_LOCK(i);
652 /* Scan fragment list. */
653 TAILQ_FOREACH_SAFE(fp, &V_ipq[i].head, ipq_list, temp) {
654 for (m = fp->ipq_frags; m != NULL; m = m->m_nextpkt) {
655 /* clear no longer valid rcvif pointer */
656 if (m->m_pkthdr.rcvif == ifp)
657 m->m_pkthdr.rcvif = NULL;
658 }
659 }
660 IPQ_UNLOCK(i);
661 }
662 CURVNET_RESTORE();
663 }
664 EVENTHANDLER_DEFINE(ifnet_departure_event, ipreass_cleanup, NULL, 0);
665
666 #ifdef VIMAGE
667 /*
668 * Destroy IP reassembly structures.
669 */
670 void
ipreass_destroy(void)671 ipreass_destroy(void)
672 {
673
674 ipreass_drain();
675 uma_zdestroy(V_ipq_zone);
676 V_ipq_zone = NULL;
677 for (int i = 0; i < IPREASS_NHASH; i++)
678 mtx_destroy(&V_ipq[i].lock);
679 }
680 #endif
681
682 /*
683 * After maxnipq has been updated, propagate the change to UMA. The UMA zone
684 * max has slightly different semantics than the sysctl, for historical
685 * reasons.
686 */
687 static void
ipreass_drain_tomax(void)688 ipreass_drain_tomax(void)
689 {
690 struct ipq *fp;
691 int target;
692
693 /*
694 * Make sure each bucket is under the new limit. If
695 * necessary, drop enough of the oldest elements from
696 * each bucket to get under the new limit.
697 */
698 for (int i = 0; i < IPREASS_NHASH; i++) {
699 IPQ_LOCK(i);
700 while (V_ipq[i].count > V_ipreass_maxbucketsize &&
701 (fp = TAILQ_LAST(&V_ipq[i].head, ipqhead)) != NULL)
702 ipq_timeout(&V_ipq[i], fp);
703 IPQ_UNLOCK(i);
704 }
705
706 /*
707 * If we are over the maximum number of fragments,
708 * drain off enough to get down to the new limit,
709 * stripping off last elements on queues. Every
710 * run we strip the oldest element from each bucket.
711 */
712 target = uma_zone_get_max(V_ipq_zone);
713 while (uma_zone_get_cur(V_ipq_zone) > target) {
714 for (int i = 0; i < IPREASS_NHASH; i++) {
715 IPQ_LOCK(i);
716 fp = TAILQ_LAST(&V_ipq[i].head, ipqhead);
717 if (fp != NULL)
718 ipq_timeout(&V_ipq[i], fp);
719 IPQ_UNLOCK(i);
720 }
721 }
722 }
723
724 static void
ipreass_zone_change(void * tag)725 ipreass_zone_change(void *tag)
726 {
727 VNET_ITERATOR_DECL(vnet_iter);
728 int max;
729
730 maxfrags = IP_MAXFRAGS;
731 max = IP_MAXFRAGPACKETS;
732 VNET_LIST_RLOCK_NOSLEEP();
733 VNET_FOREACH(vnet_iter) {
734 CURVNET_SET(vnet_iter);
735 max = uma_zone_set_max(V_ipq_zone, max);
736 V_ipreass_maxbucketsize = imax(max / (IPREASS_NHASH / 2), 1);
737 ipreass_drain_tomax();
738 CURVNET_RESTORE();
739 }
740 VNET_LIST_RUNLOCK_NOSLEEP();
741 }
742
743 /*
744 * Change the limit on the UMA zone, or disable the fragment allocation
745 * at all. Since 0 and -1 is a special values here, we need our own handler,
746 * instead of sysctl_handle_uma_zone_max().
747 */
748 static int
sysctl_maxfragpackets(SYSCTL_HANDLER_ARGS)749 sysctl_maxfragpackets(SYSCTL_HANDLER_ARGS)
750 {
751 int error, max;
752
753 if (V_noreass == 0) {
754 max = uma_zone_get_max(V_ipq_zone);
755 if (max == 0)
756 max = -1;
757 } else
758 max = 0;
759 error = sysctl_handle_int(oidp, &max, 0, req);
760 if (error || !req->newptr)
761 return (error);
762 if (max > 0) {
763 /*
764 * XXXRW: Might be a good idea to sanity check the argument
765 * and place an extreme upper bound.
766 */
767 max = uma_zone_set_max(V_ipq_zone, max);
768 V_ipreass_maxbucketsize = imax(max / (IPREASS_NHASH / 2), 1);
769 ipreass_drain_tomax();
770 V_noreass = 0;
771 } else if (max == 0) {
772 V_noreass = 1;
773 ipreass_drain();
774 } else if (max == -1) {
775 V_noreass = 0;
776 uma_zone_set_max(V_ipq_zone, 0);
777 V_ipreass_maxbucketsize = INT_MAX;
778 } else
779 return (EINVAL);
780 return (0);
781 }
782
783 /*
784 * Seek for old fragment queue header that can be reused. Try to
785 * reuse a header from currently locked hash bucket.
786 */
787 static struct ipq *
ipq_reuse(int start)788 ipq_reuse(int start)
789 {
790 struct ipq *fp;
791 int bucket, i;
792
793 IPQ_LOCK_ASSERT(start);
794
795 for (i = 0; i < IPREASS_NHASH; i++) {
796 bucket = (start + i) % IPREASS_NHASH;
797 if (bucket != start && IPQ_TRYLOCK(bucket) == 0)
798 continue;
799 fp = TAILQ_LAST(&V_ipq[bucket].head, ipqhead);
800 if (fp) {
801 struct mbuf *m;
802
803 IPSTAT_ADD(ips_fragtimeout, fp->ipq_nfrags);
804 atomic_subtract_int(&nfrags, fp->ipq_nfrags);
805 while (fp->ipq_frags) {
806 m = fp->ipq_frags;
807 fp->ipq_frags = m->m_nextpkt;
808 m_freem(m);
809 }
810 TAILQ_REMOVE(&V_ipq[bucket].head, fp, ipq_list);
811 V_ipq[bucket].count--;
812 if (bucket != start)
813 IPQ_UNLOCK(bucket);
814 break;
815 }
816 if (bucket != start)
817 IPQ_UNLOCK(bucket);
818 }
819 IPQ_LOCK_ASSERT(start);
820 return (fp);
821 }
822
823 /*
824 * Free a fragment reassembly header and all associated datagrams.
825 */
826 static void
ipq_free(struct ipqbucket * bucket,struct ipq * fp)827 ipq_free(struct ipqbucket *bucket, struct ipq *fp)
828 {
829 struct mbuf *q;
830
831 atomic_subtract_int(&nfrags, fp->ipq_nfrags);
832 while (fp->ipq_frags) {
833 q = fp->ipq_frags;
834 fp->ipq_frags = q->m_nextpkt;
835 m_freem(q);
836 }
837 TAILQ_REMOVE(&bucket->head, fp, ipq_list);
838 bucket->count--;
839 uma_zfree(V_ipq_zone, fp);
840 }
841
842 /*
843 * Get or set the maximum number of reassembly queues per bucket.
844 */
845 static int
sysctl_maxfragbucketsize(SYSCTL_HANDLER_ARGS)846 sysctl_maxfragbucketsize(SYSCTL_HANDLER_ARGS)
847 {
848 int error, max;
849
850 max = V_ipreass_maxbucketsize;
851 error = sysctl_handle_int(oidp, &max, 0, req);
852 if (error || !req->newptr)
853 return (error);
854 if (max <= 0)
855 return (EINVAL);
856 V_ipreass_maxbucketsize = max;
857 ipreass_drain_tomax();
858 return (0);
859 }
860