1 /*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/iommu.c 325900 2017-11-16 18:22:03Z jhb $
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/iommu.c 325900 2017-11-16 18:22:03Z jhb $");
31
32 #include <sys/param.h>
33 #include <sys/types.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/sysctl.h>
37
38 #include <dev/pci/pcivar.h>
39 #include <dev/pci/pcireg.h>
40
41 #include <machine/cpu.h>
42 #include <machine/md_var.h>
43
44 #include "vmm_util.h"
45 #include "vmm_mem.h"
46 #include "iommu.h"
47
48 SYSCTL_DECL(_hw_vmm);
49 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters");
50
51 static int iommu_avail;
52 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
53 0, "bhyve iommu initialized?");
54
55 static int iommu_enable = 1;
56 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
57 "Enable use of I/O MMU (required for PCI passthrough).");
58
59 static struct iommu_ops *ops;
60 static void *host_domain;
61
62 static __inline int
IOMMU_INIT(void)63 IOMMU_INIT(void)
64 {
65 if (ops != NULL)
66 return ((*ops->init)());
67 else
68 return (ENXIO);
69 }
70
71 static __inline void
IOMMU_CLEANUP(void)72 IOMMU_CLEANUP(void)
73 {
74 if (ops != NULL && iommu_avail)
75 (*ops->cleanup)();
76 }
77
78 static __inline void *
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)79 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
80 {
81
82 if (ops != NULL && iommu_avail)
83 return ((*ops->create_domain)(maxaddr));
84 else
85 return (NULL);
86 }
87
88 static __inline void
IOMMU_DESTROY_DOMAIN(void * dom)89 IOMMU_DESTROY_DOMAIN(void *dom)
90 {
91
92 if (ops != NULL && iommu_avail)
93 (*ops->destroy_domain)(dom);
94 }
95
96 static __inline uint64_t
IOMMU_CREATE_MAPPING(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)97 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
98 {
99
100 if (ops != NULL && iommu_avail)
101 return ((*ops->create_mapping)(domain, gpa, hpa, len));
102 else
103 return (len); /* XXX */
104 }
105
106 static __inline uint64_t
IOMMU_REMOVE_MAPPING(void * domain,vm_paddr_t gpa,uint64_t len)107 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
108 {
109
110 if (ops != NULL && iommu_avail)
111 return ((*ops->remove_mapping)(domain, gpa, len));
112 else
113 return (len); /* XXX */
114 }
115
116 static __inline void
IOMMU_ADD_DEVICE(void * domain,uint16_t rid)117 IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
118 {
119
120 if (ops != NULL && iommu_avail)
121 (*ops->add_device)(domain, rid);
122 }
123
124 static __inline void
IOMMU_REMOVE_DEVICE(void * domain,uint16_t rid)125 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
126 {
127
128 if (ops != NULL && iommu_avail)
129 (*ops->remove_device)(domain, rid);
130 }
131
132 static __inline void
IOMMU_INVALIDATE_TLB(void * domain)133 IOMMU_INVALIDATE_TLB(void *domain)
134 {
135
136 if (ops != NULL && iommu_avail)
137 (*ops->invalidate_tlb)(domain);
138 }
139
140 static __inline void
IOMMU_ENABLE(void)141 IOMMU_ENABLE(void)
142 {
143
144 if (ops != NULL && iommu_avail)
145 (*ops->enable)();
146 }
147
148 static __inline void
IOMMU_DISABLE(void)149 IOMMU_DISABLE(void)
150 {
151
152 if (ops != NULL && iommu_avail)
153 (*ops->disable)();
154 }
155
156 static void
iommu_init(void)157 iommu_init(void)
158 {
159 int error, bus, slot, func;
160 vm_paddr_t maxaddr;
161 devclass_t dc;
162 device_t dev;
163
164 if (!iommu_enable)
165 return;
166
167 if (vmm_is_intel())
168 ops = &iommu_ops_intel;
169 else if (vmm_is_amd())
170 ops = &iommu_ops_amd;
171 else
172 ops = NULL;
173
174 error = IOMMU_INIT();
175 if (error)
176 return;
177
178 iommu_avail = 1;
179
180 /*
181 * Create a domain for the devices owned by the host
182 */
183 maxaddr = vmm_mem_maxaddr();
184 host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
185 if (host_domain == NULL) {
186 printf("iommu_init: unable to create a host domain");
187 IOMMU_CLEANUP();
188 ops = NULL;
189 iommu_avail = 0;
190 return;
191 }
192
193 /*
194 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
195 * the host
196 */
197 iommu_create_mapping(host_domain, 0, 0, maxaddr);
198
199 dc = devclass_find("ppt");
200 for (bus = 0; bus <= PCI_BUSMAX; bus++) {
201 for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
202 for (func = 0; func <= PCI_FUNCMAX; func++) {
203 dev = pci_find_dbsf(0, bus, slot, func);
204 if (dev == NULL)
205 continue;
206
207 /* Skip passthrough devices. */
208 if (dc != NULL &&
209 device_get_devclass(dev) == dc)
210 continue;
211
212 /*
213 * Everything else belongs to the host
214 * domain.
215 */
216 iommu_add_device(host_domain,
217 pci_get_rid(dev));
218 }
219 }
220 }
221 IOMMU_ENABLE();
222
223 }
224
225 void
iommu_cleanup(void)226 iommu_cleanup(void)
227 {
228 IOMMU_DISABLE();
229 IOMMU_DESTROY_DOMAIN(host_domain);
230 IOMMU_CLEANUP();
231 }
232
233 void *
iommu_create_domain(vm_paddr_t maxaddr)234 iommu_create_domain(vm_paddr_t maxaddr)
235 {
236 static volatile int iommu_initted;
237
238 if (iommu_initted < 2) {
239 if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
240 iommu_init();
241 atomic_store_rel_int(&iommu_initted, 2);
242 } else
243 while (iommu_initted == 1)
244 cpu_spinwait();
245 }
246 return (IOMMU_CREATE_DOMAIN(maxaddr));
247 }
248
249 void
iommu_destroy_domain(void * dom)250 iommu_destroy_domain(void *dom)
251 {
252
253 IOMMU_DESTROY_DOMAIN(dom);
254 }
255
256 void
iommu_create_mapping(void * dom,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)257 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
258 {
259 uint64_t mapped, remaining;
260
261 remaining = len;
262
263 while (remaining > 0) {
264 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
265 gpa += mapped;
266 hpa += mapped;
267 remaining -= mapped;
268 }
269 }
270
271 void
iommu_remove_mapping(void * dom,vm_paddr_t gpa,size_t len)272 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
273 {
274 uint64_t unmapped, remaining;
275
276 remaining = len;
277
278 while (remaining > 0) {
279 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
280 gpa += unmapped;
281 remaining -= unmapped;
282 }
283 }
284
285 void *
iommu_host_domain(void)286 iommu_host_domain(void)
287 {
288
289 return (host_domain);
290 }
291
292 void
iommu_add_device(void * dom,uint16_t rid)293 iommu_add_device(void *dom, uint16_t rid)
294 {
295
296 IOMMU_ADD_DEVICE(dom, rid);
297 }
298
299 void
iommu_remove_device(void * dom,uint16_t rid)300 iommu_remove_device(void *dom, uint16_t rid)
301 {
302
303 IOMMU_REMOVE_DEVICE(dom, rid);
304 }
305
306 void
iommu_invalidate_tlb(void * domain)307 iommu_invalidate_tlb(void *domain)
308 {
309
310 IOMMU_INVALIDATE_TLB(domain);
311 }
312