1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD: stable/12/sys/amd64/vmm/io/iommu.c 358146 2020-02-20 01:38:56Z kib $
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: stable/12/sys/amd64/vmm/io/iommu.c 358146 2020-02-20 01:38:56Z kib $");
33
34 #include <sys/param.h>
35 #include <sys/types.h>
36 #include <sys/systm.h>
37 #include <sys/bus.h>
38 #include <sys/sysctl.h>
39
40 #include <dev/pci/pcivar.h>
41 #include <dev/pci/pcireg.h>
42
43 #include <machine/cpu.h>
44 #include <machine/md_var.h>
45
46 #include "vmm_util.h"
47 #include "vmm_mem.h"
48 #include "iommu.h"
49
50 SYSCTL_DECL(_hw_vmm);
51 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters");
52
53 static int iommu_avail;
54 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
55 0, "bhyve iommu initialized?");
56
57 static int iommu_enable = 1;
58 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
59 "Enable use of I/O MMU (required for PCI passthrough).");
60
61 static struct iommu_ops *ops;
62 static void *host_domain;
63 static eventhandler_tag add_tag, delete_tag;
64
65 static __inline int
IOMMU_INIT(void)66 IOMMU_INIT(void)
67 {
68 if (ops != NULL)
69 return ((*ops->init)());
70 else
71 return (ENXIO);
72 }
73
74 static __inline void
IOMMU_CLEANUP(void)75 IOMMU_CLEANUP(void)
76 {
77 if (ops != NULL && iommu_avail)
78 (*ops->cleanup)();
79 }
80
81 static __inline void *
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)82 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
83 {
84
85 if (ops != NULL && iommu_avail)
86 return ((*ops->create_domain)(maxaddr));
87 else
88 return (NULL);
89 }
90
91 static __inline void
IOMMU_DESTROY_DOMAIN(void * dom)92 IOMMU_DESTROY_DOMAIN(void *dom)
93 {
94
95 if (ops != NULL && iommu_avail)
96 (*ops->destroy_domain)(dom);
97 }
98
99 static __inline uint64_t
IOMMU_CREATE_MAPPING(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)100 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
101 {
102
103 if (ops != NULL && iommu_avail)
104 return ((*ops->create_mapping)(domain, gpa, hpa, len));
105 else
106 return (len); /* XXX */
107 }
108
109 static __inline uint64_t
IOMMU_REMOVE_MAPPING(void * domain,vm_paddr_t gpa,uint64_t len)110 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
111 {
112
113 if (ops != NULL && iommu_avail)
114 return ((*ops->remove_mapping)(domain, gpa, len));
115 else
116 return (len); /* XXX */
117 }
118
119 static __inline void
IOMMU_ADD_DEVICE(void * domain,uint16_t rid)120 IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
121 {
122
123 if (ops != NULL && iommu_avail)
124 (*ops->add_device)(domain, rid);
125 }
126
127 static __inline void
IOMMU_REMOVE_DEVICE(void * domain,uint16_t rid)128 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
129 {
130
131 if (ops != NULL && iommu_avail)
132 (*ops->remove_device)(domain, rid);
133 }
134
135 static __inline void
IOMMU_INVALIDATE_TLB(void * domain)136 IOMMU_INVALIDATE_TLB(void *domain)
137 {
138
139 if (ops != NULL && iommu_avail)
140 (*ops->invalidate_tlb)(domain);
141 }
142
143 static __inline void
IOMMU_ENABLE(void)144 IOMMU_ENABLE(void)
145 {
146
147 if (ops != NULL && iommu_avail)
148 (*ops->enable)();
149 }
150
151 static __inline void
IOMMU_DISABLE(void)152 IOMMU_DISABLE(void)
153 {
154
155 if (ops != NULL && iommu_avail)
156 (*ops->disable)();
157 }
158
159 static void
iommu_pci_add(void * arg,device_t dev)160 iommu_pci_add(void *arg, device_t dev)
161 {
162
163 /* Add new devices to the host domain. */
164 iommu_add_device(host_domain, pci_get_rid(dev));
165 }
166
167 static void
iommu_pci_delete(void * arg,device_t dev)168 iommu_pci_delete(void *arg, device_t dev)
169 {
170
171 iommu_remove_device(host_domain, pci_get_rid(dev));
172 }
173
174 static void
iommu_init(void)175 iommu_init(void)
176 {
177 int error, bus, slot, func;
178 vm_paddr_t maxaddr;
179 devclass_t dc;
180 device_t dev;
181
182 if (!iommu_enable)
183 return;
184
185 if (vmm_is_intel())
186 ops = &iommu_ops_intel;
187 else if (vmm_is_svm())
188 ops = &iommu_ops_amd;
189 else
190 ops = NULL;
191
192 error = IOMMU_INIT();
193 if (error)
194 return;
195
196 iommu_avail = 1;
197
198 /*
199 * Create a domain for the devices owned by the host
200 */
201 maxaddr = vmm_mem_maxaddr();
202 host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
203 if (host_domain == NULL) {
204 printf("iommu_init: unable to create a host domain");
205 IOMMU_CLEANUP();
206 ops = NULL;
207 iommu_avail = 0;
208 return;
209 }
210
211 /*
212 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
213 * the host
214 */
215 iommu_create_mapping(host_domain, 0, 0, maxaddr);
216
217 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
218 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
219 NULL, 0);
220 dc = devclass_find("ppt");
221 for (bus = 0; bus <= PCI_BUSMAX; bus++) {
222 for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
223 for (func = 0; func <= PCI_FUNCMAX; func++) {
224 dev = pci_find_dbsf(0, bus, slot, func);
225 if (dev == NULL)
226 continue;
227
228 /* Skip passthrough devices. */
229 if (dc != NULL &&
230 device_get_devclass(dev) == dc)
231 continue;
232
233 /*
234 * Everything else belongs to the host
235 * domain.
236 */
237 iommu_add_device(host_domain,
238 pci_get_rid(dev));
239 }
240 }
241 }
242 IOMMU_ENABLE();
243
244 }
245
246 void
iommu_cleanup(void)247 iommu_cleanup(void)
248 {
249
250 if (add_tag != NULL) {
251 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
252 add_tag = NULL;
253 }
254 if (delete_tag != NULL) {
255 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
256 delete_tag = NULL;
257 }
258 IOMMU_DISABLE();
259 IOMMU_DESTROY_DOMAIN(host_domain);
260 IOMMU_CLEANUP();
261 }
262
263 void *
iommu_create_domain(vm_paddr_t maxaddr)264 iommu_create_domain(vm_paddr_t maxaddr)
265 {
266 static volatile int iommu_initted;
267
268 if (iommu_initted < 2) {
269 if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
270 iommu_init();
271 atomic_store_rel_int(&iommu_initted, 2);
272 } else
273 while (iommu_initted == 1)
274 cpu_spinwait();
275 }
276 return (IOMMU_CREATE_DOMAIN(maxaddr));
277 }
278
279 void
iommu_destroy_domain(void * dom)280 iommu_destroy_domain(void *dom)
281 {
282
283 IOMMU_DESTROY_DOMAIN(dom);
284 }
285
286 void
iommu_create_mapping(void * dom,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)287 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
288 {
289 uint64_t mapped, remaining;
290
291 remaining = len;
292
293 while (remaining > 0) {
294 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
295 gpa += mapped;
296 hpa += mapped;
297 remaining -= mapped;
298 }
299 }
300
301 void
iommu_remove_mapping(void * dom,vm_paddr_t gpa,size_t len)302 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
303 {
304 uint64_t unmapped, remaining;
305
306 remaining = len;
307
308 while (remaining > 0) {
309 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
310 gpa += unmapped;
311 remaining -= unmapped;
312 }
313 }
314
315 void *
iommu_host_domain(void)316 iommu_host_domain(void)
317 {
318
319 return (host_domain);
320 }
321
322 void
iommu_add_device(void * dom,uint16_t rid)323 iommu_add_device(void *dom, uint16_t rid)
324 {
325
326 IOMMU_ADD_DEVICE(dom, rid);
327 }
328
329 void
iommu_remove_device(void * dom,uint16_t rid)330 iommu_remove_device(void *dom, uint16_t rid)
331 {
332
333 IOMMU_REMOVE_DEVICE(dom, rid);
334 }
335
336 void
iommu_invalidate_tlb(void * domain)337 iommu_invalidate_tlb(void *domain)
338 {
339
340 IOMMU_INVALIDATE_TLB(domain);
341 }
342