1 /*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31
32 __FBSDID("$FreeBSD$");
33
34 #include "opt_inet6.h"
35 #include "opt_rss.h"
36
37 #include <sys/param.h>
38 #include <sys/mbuf.h>
39 #include <sys/socket.h>
40
41 #include <net/rss_config.h>
42
43 #include <netinet/in.h>
44 #include <netinet/in_pcb.h>
45 #ifdef INET6
46 #include <netinet6/in6_pcb.h>
47 #include <netinet6/in6_rss.h>
48 #endif /* INET6 */
49
50 /*
51 * Given a hash of whatever the covered tuple might be, return a pcbgroup
52 * index. Where RSS is supported, try to align bucket selection with RSS CPU
53 * affinity strategy.
54 */
55 static __inline u_int
in6_pcbgroup_getbucket(struct inpcbinfo * pcbinfo,uint32_t hash)56 in6_pcbgroup_getbucket(struct inpcbinfo *pcbinfo, uint32_t hash)
57 {
58
59 #ifdef RSS
60 return (rss_getbucket(hash));
61 #else
62 return (hash % pcbinfo->ipi_npcbgroups);
63 #endif
64 }
65
66 /*
67 * Map a (hashtype, hash) tuple into a connection group, or NULL if the hash
68 * information is insufficient to identify the pcbgroup. This might occur if
69 * a TCP packet turnsup with a 2-tuple hash, or if an RSS hash is present but
70 * RSS is not compiled into the kernel.
71 */
72 struct inpcbgroup *
in6_pcbgroup_byhash(struct inpcbinfo * pcbinfo,u_int hashtype,uint32_t hash)73 in6_pcbgroup_byhash(struct inpcbinfo *pcbinfo, u_int hashtype, uint32_t hash)
74 {
75
76 #ifdef RSS
77 if ((pcbinfo->ipi_hashfields == IPI_HASHFIELDS_4TUPLE &&
78 hashtype == M_HASHTYPE_RSS_TCP_IPV6) ||
79 (pcbinfo->ipi_hashfields == IPI_HASHFIELDS_4TUPLE &&
80 hashtype == M_HASHTYPE_RSS_UDP_IPV6) ||
81 (pcbinfo->ipi_hashfields == IPI_HASHFIELDS_2TUPLE &&
82 hashtype == M_HASHTYPE_RSS_IPV6))
83 return (&pcbinfo->ipi_pcbgroups[
84 in6_pcbgroup_getbucket(pcbinfo, hash)]);
85 #endif
86 return (NULL);
87 }
88
89 struct inpcbgroup *
in6_pcbgroup_bymbuf(struct inpcbinfo * pcbinfo,struct mbuf * m)90 in6_pcbgroup_bymbuf(struct inpcbinfo *pcbinfo, struct mbuf *m)
91 {
92
93 return (in6_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
94 m->m_pkthdr.flowid));
95 }
96
97 struct inpcbgroup *
in6_pcbgroup_bytuple(struct inpcbinfo * pcbinfo,const struct in6_addr * laddrp,u_short lport,const struct in6_addr * faddrp,u_short fport)98 in6_pcbgroup_bytuple(struct inpcbinfo *pcbinfo, const struct in6_addr *laddrp,
99 u_short lport, const struct in6_addr *faddrp, u_short fport)
100 {
101 uint32_t hash;
102
103 /*
104 * RSS note: we pass foreign addr/port as source, and local addr/port
105 * as destination, as we want to align with what the hardware is
106 * doing.
107 */
108 switch (pcbinfo->ipi_hashfields) {
109 case IPI_HASHFIELDS_4TUPLE:
110 #ifdef RSS
111 hash = rss_hash_ip6_4tuple(faddrp, fport, laddrp, lport);
112 #else
113 hash = faddrp->s6_addr32[3] ^ fport;
114 #endif
115 break;
116
117 case IPI_HASHFIELDS_2TUPLE:
118 #ifdef RSS
119 hash = rss_hash_ip6_2tuple(faddrp, laddrp);
120 #else
121 hash = faddrp->s6_addr32[3] ^ laddrp->s6_addr32[3];
122 #endif
123 break;
124
125 default:
126 hash = 0;
127 }
128 return (&pcbinfo->ipi_pcbgroups[in6_pcbgroup_getbucket(pcbinfo,
129 hash)]);
130 }
131
132 struct inpcbgroup *
in6_pcbgroup_byinpcb(struct inpcb * inp)133 in6_pcbgroup_byinpcb(struct inpcb *inp)
134 {
135
136 #ifdef RSS
137 /*
138 * Listen sockets with INP_RSS_BUCKET_SET set have a pre-determined
139 * RSS bucket and thus we should use this pcbgroup, rather than
140 * using a tuple or hash.
141 *
142 * XXX should verify that there's actually pcbgroups and inp_rss_listen_bucket
143 * fits in that!
144 */
145 if (inp->inp_flags2 & INP_RSS_BUCKET_SET)
146 return (&inp->inp_pcbinfo->ipi_pcbgroups[inp->inp_rss_listen_bucket]);
147 #endif
148
149 return (in6_pcbgroup_bytuple(inp->inp_pcbinfo, &inp->in6p_laddr,
150 inp->inp_lport, &inp->in6p_faddr, inp->inp_fport));
151 }
152