1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/bus.h>
33 #include <sys/kernel.h>
34 #include <sys/module.h>
35 #include <sys/malloc.h>
36 #include <sys/pcpu.h>
37 #include <sys/rman.h>
38 #include <sys/smp.h>
39 #include <sys/sysctl.h>
40
41 #include <vm/vm.h>
42 #include <vm/pmap.h>
43
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
46
47 #include <machine/resource.h>
48 #include <machine/vmm.h>
49 #include <machine/pmap.h>
50 #include <machine/vmparam.h>
51 #include <machine/pci_cfgreg.h>
52
53 #include "ivhd_if.h"
54 #include "pcib_if.h"
55
56 #include "io/iommu.h"
57 #include "amdvi_priv.h"
58
59 SYSCTL_DECL(_hw_vmm);
60 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
61 NULL);
62
63 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
64 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
65
66 /* Print RID or device ID in PCI string format. */
67 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
68
69 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count);
70 static void amdvi_print_dev_cap(struct amdvi_softc *softc);
71
72 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
73
74 extern device_t *ivhd_devs;
75
76 extern int ivhd_count;
77 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
78 0, NULL);
79
80 static int amdvi_enable_user = 0;
81 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
82 &amdvi_enable_user, 0, NULL);
83 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
84
85 #ifdef AMDVI_ATS_ENABLE
86 /* XXX: ATS is not tested. */
87 static int amdvi_enable_iotlb = 1;
88 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
89 &amdvi_enable_iotlb, 0, NULL);
90 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
91 #endif
92
93 static int amdvi_host_ptp = 1; /* Use page tables for host. */
94 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
95 &amdvi_host_ptp, 0, NULL);
96 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
97
98 /* Page table level used <= supported by h/w[v1=7]. */
99 int amdvi_ptp_level = 4;
100 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
101 &amdvi_ptp_level, 0, NULL);
102 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
103
104 /* Disable fault event reporting. */
105 static int amdvi_disable_io_fault = 0;
106 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
107 &amdvi_disable_io_fault, 0, NULL);
108 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
109
110 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */
111 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
112 &amdvi_dom_id, 0, NULL);
113 /*
114 * Device table entry.
115 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
116 * = 256 * 2 * PAGE_SIZE.
117 */
118 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
119 CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
120 CTASSERT(sizeof(amdvi_dte) == 0x200000);
121
122 static SLIST_HEAD (, amdvi_domain) dom_head;
123
124 static inline uint32_t
amdvi_pci_read(struct amdvi_softc * softc,int off)125 amdvi_pci_read(struct amdvi_softc *softc, int off)
126 {
127
128 return (pci_cfgregread(softc->pci_seg, PCI_RID2BUS(softc->pci_rid),
129 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
130 off, 4));
131 }
132
133 #ifdef AMDVI_ATS_ENABLE
134 /* XXX: Should be in pci.c */
135 /*
136 * Check if device has ATS capability and its enabled.
137 * If ATS is absent or disabled, return (-1), otherwise ATS
138 * queue length.
139 */
140 static int
amdvi_find_ats_qlen(uint16_t devid)141 amdvi_find_ats_qlen(uint16_t devid)
142 {
143 device_t dev;
144 uint32_t off, cap;
145 int qlen = -1;
146
147 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
148 PCI_RID2FUNC(devid));
149
150 if (!dev) {
151 return (-1);
152 }
153 #define PCIM_ATS_EN BIT(31)
154
155 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
156 cap = pci_read_config(dev, off + 4, 4);
157 qlen = (cap & 0x1F);
158 qlen = qlen ? qlen : 32;
159 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
160 RID2PCI_STR(devid),
161 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
162 qlen);
163 qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
164 }
165
166 return (qlen);
167 }
168
169 /*
170 * Check if an endpoint device support device IOTLB or ATS.
171 */
172 static inline bool
amdvi_dev_support_iotlb(struct amdvi_softc * softc,uint16_t devid)173 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
174 {
175 struct ivhd_dev_cfg *cfg;
176 int qlen, i;
177 bool pci_ats, ivhd_ats;
178
179 qlen = amdvi_find_ats_qlen(devid);
180 if (qlen < 0)
181 return (false);
182
183 KASSERT(softc, ("softc is NULL"));
184 cfg = softc->dev_cfg;
185
186 ivhd_ats = false;
187 for (i = 0; i < softc->dev_cfg_cnt; i++) {
188 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
189 ivhd_ats = cfg->enable_ats;
190 break;
191 }
192 cfg++;
193 }
194
195 pci_ats = (qlen < 0) ? false : true;
196 if (pci_ats != ivhd_ats)
197 device_printf(softc->dev,
198 "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
199 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
200
201 /* Ignore IVRS setting and respect PCI setting. */
202 return (pci_ats);
203 }
204 #endif
205
206 /* Enable IOTLB support for IOMMU if its supported. */
207 static inline void
amdvi_hw_enable_iotlb(struct amdvi_softc * softc)208 amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
209 {
210 #ifndef AMDVI_ATS_ENABLE
211 softc->iotlb = false;
212 #else
213 bool supported;
214
215 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
216
217 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
218 if (!supported)
219 device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
220
221 if (supported && !amdvi_enable_iotlb) {
222 device_printf(softc->dev, "IOTLB disabled by user.\n");
223 supported = false;
224 }
225 } else
226 supported = false;
227
228 softc->iotlb = supported;
229
230 #endif
231 }
232
233 static int
amdvi_init_cmd(struct amdvi_softc * softc)234 amdvi_init_cmd(struct amdvi_softc *softc)
235 {
236 struct amdvi_ctrl *ctrl = softc->ctrl;
237
238 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */
239 softc->cmd_max = 1 << ctrl->cmd.len;
240
241 softc->cmd = malloc(sizeof(struct amdvi_cmd) *
242 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
243
244 if ((uintptr_t)softc->cmd & PAGE_MASK)
245 panic("AMDVi: Command buffer not aligned on page boundary.");
246
247 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
248 /*
249 * XXX: Reset the h/w pointers in case IOMMU is restarting,
250 * h/w doesn't clear these pointers based on empirical data.
251 */
252 ctrl->cmd_tail = 0;
253 ctrl->cmd_head = 0;
254
255 return (0);
256 }
257
258 /*
259 * Note: Update tail pointer after we have written the command since tail
260 * pointer update cause h/w to execute new commands, see section 3.3
261 * of AMD IOMMU spec ver 2.0.
262 */
263 /* Get the command tail pointer w/o updating it. */
264 static struct amdvi_cmd *
amdvi_get_cmd_tail(struct amdvi_softc * softc)265 amdvi_get_cmd_tail(struct amdvi_softc *softc)
266 {
267 struct amdvi_ctrl *ctrl;
268 struct amdvi_cmd *tail;
269
270 KASSERT(softc, ("softc is NULL"));
271 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
272
273 ctrl = softc->ctrl;
274 KASSERT(ctrl != NULL, ("ctrl is NULL"));
275
276 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
277 ctrl->cmd_tail);
278
279 return (tail);
280 }
281
282 /*
283 * Update the command tail pointer which will start command execution.
284 */
285 static void
amdvi_update_cmd_tail(struct amdvi_softc * softc)286 amdvi_update_cmd_tail(struct amdvi_softc *softc)
287 {
288 struct amdvi_ctrl *ctrl;
289 int size;
290
291 size = sizeof(struct amdvi_cmd);
292 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
293
294 ctrl = softc->ctrl;
295 KASSERT(ctrl != NULL, ("ctrl is NULL"));
296
297 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
298 softc->total_cmd++;
299
300 #ifdef AMDVI_DEBUG_CMD
301 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
302 ctrl->cmd_tail,
303 ctrl->cmd_head);
304 #endif
305
306 }
307
308 /*
309 * Various commands supported by IOMMU.
310 */
311
312 /* Completion wait command. */
313 static void
amdvi_cmd_cmp(struct amdvi_softc * softc,const uint64_t data)314 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
315 {
316 struct amdvi_cmd *cmd;
317 uint64_t pa;
318
319 cmd = amdvi_get_cmd_tail(softc);
320 KASSERT(cmd != NULL, ("Cmd is NULL"));
321
322 pa = vtophys(&softc->cmp_data);
323 cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
324 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
325 cmd->word1 = (pa >> 32) & 0xFFFFF;
326 cmd->addr = data;
327
328 amdvi_update_cmd_tail(softc);
329 }
330
331 /* Invalidate device table entry. */
332 static void
amdvi_cmd_inv_dte(struct amdvi_softc * softc,uint16_t devid)333 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
334 {
335 struct amdvi_cmd *cmd;
336
337 cmd = amdvi_get_cmd_tail(softc);
338 KASSERT(cmd != NULL, ("Cmd is NULL"));
339 cmd->opcode = AMDVI_INVD_DTE_OPCODE;
340 cmd->word0 = devid;
341 amdvi_update_cmd_tail(softc);
342 #ifdef AMDVI_DEBUG_CMD
343 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
344 #endif
345 }
346
347 /* Invalidate IOMMU page, use for invalidation of domain. */
348 static void
amdvi_cmd_inv_iommu_pages(struct amdvi_softc * softc,uint16_t domain_id,uint64_t addr,bool guest_nested,bool pde,bool page)349 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
350 uint64_t addr, bool guest_nested,
351 bool pde, bool page)
352 {
353 struct amdvi_cmd *cmd;
354
355 cmd = amdvi_get_cmd_tail(softc);
356 KASSERT(cmd != NULL, ("Cmd is NULL"));
357
358 cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
359 cmd->word1 = domain_id;
360 /*
361 * Invalidate all addresses for this domain.
362 */
363 cmd->addr = addr;
364 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
365 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
366
367 amdvi_update_cmd_tail(softc);
368 }
369
370 #ifdef AMDVI_ATS_ENABLE
371 /* Invalidate device IOTLB. */
372 static void
amdvi_cmd_inv_iotlb(struct amdvi_softc * softc,uint16_t devid)373 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
374 {
375 struct amdvi_cmd *cmd;
376 int qlen;
377
378 if (!softc->iotlb)
379 return;
380
381 qlen = amdvi_find_ats_qlen(devid);
382 if (qlen < 0) {
383 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
384 qlen, RID2PCI_STR(devid));
385 }
386 cmd = amdvi_get_cmd_tail(softc);
387 KASSERT(cmd != NULL, ("Cmd is NULL"));
388
389 #ifdef AMDVI_DEBUG_CMD
390 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
391 " Qlen:%d\n", devid, qlen);
392 #endif
393 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
394 cmd->word0 = devid;
395 cmd->word1 = qlen;
396 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
397 AMDVI_INVD_IOTLB_S;
398 amdvi_update_cmd_tail(softc);
399 }
400 #endif
401
402 #ifdef notyet /* For Interrupt Remap. */
403 static void
amdvi_cmd_inv_intr_map(struct amdvi_softc * softc,uint16_t devid)404 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
405 uint16_t devid)
406 {
407 struct amdvi_cmd *cmd;
408
409 cmd = amdvi_get_cmd_tail(softc);
410 KASSERT(cmd != NULL, ("Cmd is NULL"));
411 cmd->opcode = AMDVI_INVD_INTR_OPCODE;
412 cmd->word0 = devid;
413 amdvi_update_cmd_tail(softc);
414 #ifdef AMDVI_DEBUG_CMD
415 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
416 #endif
417 }
418 #endif
419
420 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
421 static void
amdvi_inv_domain(struct amdvi_softc * softc,uint16_t domain_id)422 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
423 {
424 struct amdvi_cmd *cmd __diagused;
425
426 cmd = amdvi_get_cmd_tail(softc);
427 KASSERT(cmd != NULL, ("Cmd is NULL"));
428
429 /*
430 * See section 3.3.3 of IOMMU spec rev 2.0, software note
431 * for invalidating domain.
432 */
433 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
434 false, true, true);
435
436 #ifdef AMDVI_DEBUG_CMD
437 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
438
439 #endif
440 }
441
442 static bool
amdvi_cmp_wait(struct amdvi_softc * softc)443 amdvi_cmp_wait(struct amdvi_softc *softc)
444 {
445 #ifdef AMDVI_DEBUG_CMD
446 struct amdvi_ctrl *ctrl = softc->ctrl;
447 #endif
448 const uint64_t VERIFY = 0xA5A5;
449 volatile uint64_t *read;
450 int i;
451 bool status;
452
453 read = &softc->cmp_data;
454 *read = 0;
455 amdvi_cmd_cmp(softc, VERIFY);
456 /* Wait for h/w to update completion data. */
457 for (i = 0; i < 100 && (*read != VERIFY); i++) {
458 DELAY(1000); /* 1 ms */
459 }
460 status = (VERIFY == softc->cmp_data) ? true : false;
461
462 #ifdef AMDVI_DEBUG_CMD
463 if (status)
464 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
465 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
466 ctrl->cmd_head, loop);
467 #endif
468 return (status);
469 }
470
471 static void
amdvi_wait(struct amdvi_softc * softc)472 amdvi_wait(struct amdvi_softc *softc)
473 {
474 struct amdvi_ctrl *ctrl;
475 int i;
476
477 KASSERT(softc, ("softc is NULL"));
478
479 ctrl = softc->ctrl;
480 KASSERT(ctrl != NULL, ("ctrl is NULL"));
481 /* Don't wait if h/w is not enabled. */
482 if ((ctrl->control & AMDVI_CTRL_EN) == 0)
483 return;
484
485 for (i = 0; i < 10; i++) {
486 if (amdvi_cmp_wait(softc))
487 return;
488 }
489
490 device_printf(softc->dev, "Error: completion failed"
491 " tail:0x%x, head:0x%x.\n",
492 ctrl->cmd_tail, ctrl->cmd_head);
493 /* Dump the last command. */
494 amdvi_dump_cmds(softc, 1);
495 }
496
497 static void
amdvi_dump_cmds(struct amdvi_softc * softc,int count)498 amdvi_dump_cmds(struct amdvi_softc *softc, int count)
499 {
500 struct amdvi_ctrl *ctrl;
501 struct amdvi_cmd *cmd;
502 int off, i;
503
504 ctrl = softc->ctrl;
505 device_printf(softc->dev, "Dump last %d command(s):\n", count);
506 /*
507 * If h/w is stuck in completion, it is the previous command,
508 * start dumping from previous command onward.
509 */
510 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
511 softc->cmd_max);
512 for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
513 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
514 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
515 " 0x%x 0x%lx\n", i, off, cmd->opcode,
516 cmd->word0, cmd->word1, cmd->addr);
517 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
518 }
519 }
520
521 static int
amdvi_init_event(struct amdvi_softc * softc)522 amdvi_init_event(struct amdvi_softc *softc)
523 {
524 struct amdvi_ctrl *ctrl;
525
526 ctrl = softc->ctrl;
527 ctrl->event.len = 8;
528 softc->event_max = 1 << ctrl->event.len;
529 softc->event = malloc(sizeof(struct amdvi_event) *
530 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
531 if ((uintptr_t)softc->event & PAGE_MASK) {
532 device_printf(softc->dev, "Event buffer not aligned on page.");
533 return (false);
534 }
535 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
536
537 /* Reset the pointers. */
538 ctrl->evt_head = 0;
539 ctrl->evt_tail = 0;
540
541 return (0);
542 }
543
544 static inline void
amdvi_decode_evt_flag(uint16_t flag)545 amdvi_decode_evt_flag(uint16_t flag)
546 {
547
548 flag &= AMDVI_EVENT_FLAG_MASK;
549 printf(" 0x%b]\n", flag,
550 "\020"
551 "\001GN"
552 "\002NX"
553 "\003US"
554 "\004I"
555 "\005PR"
556 "\006RW"
557 "\007PE"
558 "\010RZ"
559 "\011TR"
560 );
561 }
562
563 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
564 static inline void
amdvi_decode_evt_flag_type(uint8_t type)565 amdvi_decode_evt_flag_type(uint8_t type)
566 {
567
568 switch (AMDVI_EVENT_FLAG_TYPE(type)) {
569 case 0:
570 printf("RSVD\n");
571 break;
572 case 1:
573 printf("Master Abort\n");
574 break;
575 case 2:
576 printf("Target Abort\n");
577 break;
578 case 3:
579 printf("Data Err\n");
580 break;
581 default:
582 break;
583 }
584 }
585
586 static void
amdvi_decode_inv_dte_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)587 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
588 uint16_t flag)
589 {
590
591 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
592 " Addr:0x%lx",
593 devid, domid, addr);
594 amdvi_decode_evt_flag(flag);
595 }
596
597 static void
amdvi_decode_pf_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)598 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
599 uint16_t flag)
600 {
601
602 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
603 " Addr:0x%lx",
604 devid, domid, addr);
605 amdvi_decode_evt_flag(flag);
606 }
607
608 static void
amdvi_decode_dte_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)609 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
610 uint64_t addr, uint16_t flag)
611 {
612
613 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
614 " Addr:0x%lx", devid, domid, addr);
615 amdvi_decode_evt_flag(flag);
616 amdvi_decode_evt_flag_type(flag);
617 }
618
619 static void
amdvi_decode_page_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)620 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
621 uint16_t flag)
622 {
623
624 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
625 " Addr:0x%lx", devid, domid, addr);
626 amdvi_decode_evt_flag(flag);
627 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
628 }
629
630 static void
amdvi_decode_evt(struct amdvi_event * evt)631 amdvi_decode_evt(struct amdvi_event *evt)
632 {
633 struct amdvi_cmd *cmd;
634
635 switch (evt->opcode) {
636 case AMDVI_EVENT_INVALID_DTE:
637 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
638 evt->addr, evt->flag);
639 break;
640
641 case AMDVI_EVENT_PFAULT:
642 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
643 evt->addr, evt->flag);
644 break;
645
646 case AMDVI_EVENT_DTE_HW_ERROR:
647 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
648 evt->addr, evt->flag);
649 break;
650
651 case AMDVI_EVENT_PAGE_HW_ERROR:
652 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
653 evt->addr, evt->flag);
654 break;
655
656 case AMDVI_EVENT_ILLEGAL_CMD:
657 /* FALL THROUGH */
658 case AMDVI_EVENT_CMD_HW_ERROR:
659 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
660 "ILLEGAL CMD" : "CMD HW ERR");
661 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
662 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
663 cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
664 break;
665
666 case AMDVI_EVENT_IOTLB_TIMEOUT:
667 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
668 evt->devid, evt->addr);
669 break;
670
671 case AMDVI_EVENT_INVALID_DTE_REQ:
672 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
673 evt->devid, evt->addr, evt->flag >> 9,
674 (evt->flag >> 8) & 1);
675 break;
676
677 case AMDVI_EVENT_INVALID_PPR_REQ:
678 case AMDVI_EVENT_COUNTER_ZERO:
679 printf("AMD-Vi: v2 events.\n");
680 break;
681
682 default:
683 printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
684 }
685 }
686
687 static void
amdvi_print_events(struct amdvi_softc * softc)688 amdvi_print_events(struct amdvi_softc *softc)
689 {
690 struct amdvi_ctrl *ctrl;
691 struct amdvi_event *event;
692 int i, size;
693
694 ctrl = softc->ctrl;
695 size = sizeof(struct amdvi_event);
696 for (i = 0; i < softc->event_max; i++) {
697 event = &softc->event[ctrl->evt_head / size];
698 if (!event->opcode)
699 break;
700 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
701 i, ctrl->evt_head, ctrl->evt_tail);
702 amdvi_decode_evt(event);
703 ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
704 softc->event_max);
705 }
706 }
707
708 static int
amdvi_init_dte(struct amdvi_softc * softc)709 amdvi_init_dte(struct amdvi_softc *softc)
710 {
711 struct amdvi_ctrl *ctrl;
712
713 ctrl = softc->ctrl;
714 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
715 ctrl->dte.size = 0x1FF; /* 2MB device table. */
716
717 return (0);
718 }
719
720 /*
721 * Not all capabilities of IOMMU are available in ACPI IVHD flag
722 * or EFR entry, read directly from device.
723 */
724 static int
amdvi_print_pci_cap(device_t dev)725 amdvi_print_pci_cap(device_t dev)
726 {
727 struct amdvi_softc *softc;
728 uint32_t off, cap;
729
730 softc = device_get_softc(dev);
731 off = softc->cap_off;
732
733 /*
734 * Section 3.7.1 of IOMMU sepc rev 2.0.
735 * Read capability from device.
736 */
737 cap = amdvi_pci_read(softc, off);
738
739 /* Make sure capability type[18:16] is 3. */
740 KASSERT((((cap >> 16) & 0x7) == 0x3),
741 ("Not a IOMMU capability 0x%x@0x%x", cap, off));
742
743 softc->pci_cap = cap >> 24;
744 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
745 cap, off, softc->pci_cap,
746 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
747
748 return (0);
749 }
750
751 static void
amdvi_event_intr(void * arg)752 amdvi_event_intr(void *arg)
753 {
754 struct amdvi_softc *softc;
755 struct amdvi_ctrl *ctrl;
756
757 softc = (struct amdvi_softc *)arg;
758 ctrl = softc->ctrl;
759 device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
760 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
761 ctrl->status, ctrl->evt_head, ctrl->evt_tail);
762 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
763 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
764
765 amdvi_print_events(softc);
766 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
767 }
768
769 static void
amdvi_free_evt_intr_res(device_t dev)770 amdvi_free_evt_intr_res(device_t dev)
771 {
772
773 struct amdvi_softc *softc;
774 device_t mmio_dev;
775
776 softc = device_get_softc(dev);
777 mmio_dev = softc->pci_dev;
778
779 IVHD_TEARDOWN_INTR(mmio_dev);
780 }
781
782 static bool
amdvi_alloc_intr_resources(struct amdvi_softc * softc)783 amdvi_alloc_intr_resources(struct amdvi_softc *softc)
784 {
785 struct amdvi_ctrl *ctrl;
786 device_t dev, mmio_dev;
787 int err;
788
789 dev = softc->dev;
790 mmio_dev = softc->pci_dev;
791
792 /* Clear interrupt status bits. */
793 ctrl = softc->ctrl;
794 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
795
796 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault");
797 if (err)
798 device_printf(dev, "Interrupt setup failed on %s\n",
799 device_get_nameunit(mmio_dev));
800 return (err);
801 }
802
803 static void
amdvi_print_dev_cap(struct amdvi_softc * softc)804 amdvi_print_dev_cap(struct amdvi_softc *softc)
805 {
806 struct ivhd_dev_cfg *cfg;
807 int i;
808
809 cfg = softc->dev_cfg;
810 for (i = 0; i < softc->dev_cfg_cnt; i++) {
811 device_printf(softc->dev, "device [0x%x - 0x%x] "
812 "config:%b%s\n", cfg->start_id, cfg->end_id,
813 cfg->data,
814 "\020\001INIT\002ExtInt\003NMI"
815 "\007LINT0\010LINT1",
816 cfg->enable_ats ? "ATS enabled" : "");
817 cfg++;
818 }
819 }
820
821 static int
amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)822 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
823 {
824 struct amdvi_softc *softc;
825 int result, type, error = 0;
826
827 softc = (struct amdvi_softc *)arg1;
828 type = arg2;
829
830 switch (type) {
831 case 0:
832 result = softc->ctrl->cmd_head;
833 error = sysctl_handle_int(oidp, &result, 0,
834 req);
835 break;
836 case 1:
837 result = softc->ctrl->cmd_tail;
838 error = sysctl_handle_int(oidp, &result, 0,
839 req);
840 break;
841 case 2:
842 result = softc->ctrl->evt_head;
843 error = sysctl_handle_int(oidp, &result, 0,
844 req);
845 break;
846 case 3:
847 result = softc->ctrl->evt_tail;
848 error = sysctl_handle_int(oidp, &result, 0,
849 req);
850 break;
851
852 default:
853 device_printf(softc->dev, "Unknown sysctl:%d\n", type);
854 }
855
856 return (error);
857 }
858
859 static void
amdvi_add_sysctl(struct amdvi_softc * softc)860 amdvi_add_sysctl(struct amdvi_softc *softc)
861 {
862 struct sysctl_oid_list *child;
863 struct sysctl_ctx_list *ctx;
864 device_t dev;
865
866 dev = softc->dev;
867 ctx = device_get_sysctl_ctx(dev);
868 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
869
870 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
871 &softc->event_intr_cnt, "Event interrupt count");
872 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
873 &softc->total_cmd, "Command submitted count");
874 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
875 &softc->pci_rid, 0, "IOMMU RID");
876 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
877 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0,
878 amdvi_handle_sysctl, "IU", "Command head");
879 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
880 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1,
881 amdvi_handle_sysctl, "IU", "Command tail");
882 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
883 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2,
884 amdvi_handle_sysctl, "IU", "Command head");
885 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
886 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3,
887 amdvi_handle_sysctl, "IU", "Command tail");
888 }
889
890 int
amdvi_setup_hw(struct amdvi_softc * softc)891 amdvi_setup_hw(struct amdvi_softc *softc)
892 {
893 device_t dev;
894 int status;
895
896 dev = softc->dev;
897
898 amdvi_hw_enable_iotlb(softc);
899
900 amdvi_print_dev_cap(softc);
901
902 if ((status = amdvi_print_pci_cap(dev)) != 0) {
903 device_printf(dev, "PCI capability.\n");
904 return (status);
905 }
906 if ((status = amdvi_init_cmd(softc)) != 0) {
907 device_printf(dev, "Couldn't configure command buffer.\n");
908 return (status);
909 }
910 if ((status = amdvi_init_event(softc)) != 0) {
911 device_printf(dev, "Couldn't configure event buffer.\n");
912 return (status);
913 }
914 if ((status = amdvi_init_dte(softc)) != 0) {
915 device_printf(dev, "Couldn't configure device table.\n");
916 return (status);
917 }
918 if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
919 return (status);
920 }
921 amdvi_add_sysctl(softc);
922 return (0);
923 }
924
925 int
amdvi_teardown_hw(struct amdvi_softc * softc)926 amdvi_teardown_hw(struct amdvi_softc *softc)
927 {
928 device_t dev;
929
930 dev = softc->dev;
931
932 /*
933 * Called after disable, h/w is stopped by now, free all the resources.
934 */
935 amdvi_free_evt_intr_res(dev);
936
937 if (softc->cmd)
938 free(softc->cmd, M_AMDVI);
939
940 if (softc->event)
941 free(softc->event, M_AMDVI);
942
943 return (0);
944 }
945
946 /*********** bhyve interfaces *********************/
947 static int
amdvi_init(void)948 amdvi_init(void)
949 {
950 if (!ivhd_count) {
951 return (EIO);
952 }
953 if (!amdvi_enable_user && ivhd_count) {
954 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
955 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
956 ivhd_count);
957 return (EINVAL);
958 }
959 return (0);
960 }
961
962 static void
amdvi_cleanup(void)963 amdvi_cleanup(void)
964 {
965 /* Nothing. */
966 }
967
968 static uint16_t
amdvi_domainId(void)969 amdvi_domainId(void)
970 {
971
972 /*
973 * If we hit maximum domain limit, rollover leaving host
974 * domain(0).
975 * XXX: make sure that this domain is not used.
976 */
977 if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
978 amdvi_dom_id = 1;
979
980 return ((uint16_t)amdvi_dom_id++);
981 }
982
983 static void
amdvi_do_inv_domain(uint16_t domain_id,bool create)984 amdvi_do_inv_domain(uint16_t domain_id, bool create)
985 {
986 struct amdvi_softc *softc;
987 int i;
988
989 for (i = 0; i < ivhd_count; i++) {
990 softc = device_get_softc(ivhd_devs[i]);
991 KASSERT(softc, ("softc is NULL"));
992 /*
993 * If not present pages are cached, invalidate page after
994 * creating domain.
995 */
996 #if 0
997 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
998 continue;
999 #endif
1000 amdvi_inv_domain(softc, domain_id);
1001 amdvi_wait(softc);
1002 }
1003 }
1004
1005 static void *
amdvi_create_domain(vm_paddr_t maxaddr)1006 amdvi_create_domain(vm_paddr_t maxaddr)
1007 {
1008 struct amdvi_domain *dom;
1009
1010 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
1011 dom->id = amdvi_domainId();
1012 //dom->maxaddr = maxaddr;
1013 #ifdef AMDVI_DEBUG_CMD
1014 printf("Created domain #%d\n", dom->id);
1015 #endif
1016 /*
1017 * Host domain(#0) don't create translation table.
1018 */
1019 if (dom->id || amdvi_host_ptp)
1020 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1021
1022 dom->ptp_level = amdvi_ptp_level;
1023
1024 amdvi_do_inv_domain(dom->id, true);
1025 SLIST_INSERT_HEAD(&dom_head, dom, next);
1026
1027 return (dom);
1028 }
1029
1030 static void
amdvi_free_ptp(uint64_t * ptp,int level)1031 amdvi_free_ptp(uint64_t *ptp, int level)
1032 {
1033 int i;
1034
1035 if (level < 1)
1036 return;
1037
1038 for (i = 0; i < NPTEPG ; i++) {
1039 if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
1040 continue;
1041 /* XXX: Add super-page or PTE mapping > 4KB. */
1042 #ifdef notyet
1043 /* Super-page mapping. */
1044 if (AMDVI_PD_SUPER(ptp[i]))
1045 continue;
1046 #endif
1047
1048 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
1049 & AMDVI_PT_MASK), level - 1);
1050 }
1051
1052 free(ptp, M_AMDVI);
1053 }
1054
1055 static void
amdvi_destroy_domain(void * arg)1056 amdvi_destroy_domain(void *arg)
1057 {
1058 struct amdvi_domain *domain;
1059
1060 domain = (struct amdvi_domain *)arg;
1061 KASSERT(domain, ("domain is NULL"));
1062 #ifdef AMDVI_DEBUG_CMD
1063 printf("Destroying domain %d\n", domain->id);
1064 #endif
1065 if (domain->ptp)
1066 amdvi_free_ptp(domain->ptp, domain->ptp_level);
1067
1068 amdvi_do_inv_domain(domain->id, false);
1069 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
1070 free(domain, M_AMDVI);
1071 }
1072
1073 static uint64_t
amdvi_set_pt(uint64_t * pt,int level,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t pg_size,bool create)1074 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
1075 vm_paddr_t hpa, uint64_t pg_size, bool create)
1076 {
1077 uint64_t *page, pa;
1078 int shift, index;
1079 const int PT_SHIFT = 9;
1080 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */
1081
1082 if (!pg_size)
1083 return (0);
1084
1085 if (hpa & (pg_size - 1)) {
1086 printf("HPA is not size aligned.\n");
1087 return (0);
1088 }
1089 if (gpa & (pg_size - 1)) {
1090 printf("HPA is not size aligned.\n");
1091 return (0);
1092 }
1093 shift = PML4SHIFT;
1094 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
1095 index = (gpa >> shift) & PT_INDEX_MASK;
1096
1097 if ((pt[index] == 0) && create) {
1098 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1099 pa = vtophys(page);
1100 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
1101 ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1102 }
1103 #ifdef AMDVI_DEBUG_PTE
1104 if ((gpa % 0x1000000) == 0)
1105 printf("[level%d, shift = %d]PTE:0x%lx\n",
1106 level, shift, pt[index]);
1107 #endif
1108 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK)
1109 pa = PTE2PA(pt[index]);
1110 pt = (uint64_t *)PHYS_TO_DMAP(pa);
1111 shift -= PT_SHIFT;
1112 level--;
1113 }
1114
1115 /* Leaf entry. */
1116 index = (gpa >> shift) & PT_INDEX_MASK;
1117
1118 if (create) {
1119 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
1120 } else
1121 pt[index] = 0;
1122
1123 #ifdef AMDVI_DEBUG_PTE
1124 if ((gpa % 0x1000000) == 0)
1125 printf("[Last level%d, shift = %d]PTE:0x%lx\n",
1126 level, shift, pt[index]);
1127 #endif
1128 return (1ULL << shift);
1129 }
1130
1131 static uint64_t
amdvi_update_mapping(struct amdvi_domain * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t size,bool create)1132 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
1133 vm_paddr_t hpa, uint64_t size, bool create)
1134 {
1135 uint64_t mapped, *ptp, len;
1136 int level;
1137
1138 KASSERT(domain, ("domain is NULL"));
1139 level = domain->ptp_level;
1140 KASSERT(level, ("Page table level is 0"));
1141
1142 ptp = domain->ptp;
1143 KASSERT(ptp, ("PTP is NULL"));
1144 mapped = 0;
1145 while (mapped < size) {
1146 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1147 PAGE_SIZE, create);
1148 if (!len) {
1149 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
1150 hpa, gpa);
1151 return (0);
1152 }
1153 mapped += len;
1154 }
1155
1156 return (mapped);
1157 }
1158
1159 static uint64_t
amdvi_create_mapping(void * arg,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)1160 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
1161 uint64_t len)
1162 {
1163 struct amdvi_domain *domain;
1164
1165 domain = (struct amdvi_domain *)arg;
1166
1167 if (domain->id && !domain->ptp) {
1168 printf("ptp is NULL");
1169 return (-1);
1170 }
1171
1172 /*
1173 * If host domain is created w/o page table, skip IOMMU page
1174 * table set-up.
1175 */
1176 if (domain->ptp)
1177 return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1178 else
1179 return (len);
1180 }
1181
1182 static uint64_t
amdvi_remove_mapping(void * arg,vm_paddr_t gpa,uint64_t len)1183 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1184 {
1185 struct amdvi_domain *domain;
1186
1187 domain = (struct amdvi_domain *)arg;
1188 /*
1189 * If host domain is created w/o page table, skip IOMMU page
1190 * table set-up.
1191 */
1192 if (domain->ptp)
1193 return (amdvi_update_mapping(domain, gpa, 0, len, false));
1194 return
1195 (len);
1196 }
1197
1198 static struct amdvi_softc *
amdvi_find_iommu(uint16_t devid)1199 amdvi_find_iommu(uint16_t devid)
1200 {
1201 struct amdvi_softc *softc;
1202 int i, j;
1203
1204 for (i = 0; i < ivhd_count; i++) {
1205 softc = device_get_softc(ivhd_devs[i]);
1206 for (j = 0; j < softc->dev_cfg_cnt; j++)
1207 if ((devid >= softc->dev_cfg[j].start_id) &&
1208 (devid <= softc->dev_cfg[j].end_id))
1209 return (softc);
1210 }
1211
1212 return (NULL);
1213 }
1214
1215 /*
1216 * Set-up device table entry.
1217 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1218 * be set concurrently, e.g. read and write bits.
1219 */
1220 static void
amdvi_set_dte(struct amdvi_domain * domain,struct amdvi_softc * softc,uint16_t devid,bool enable)1221 amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc,
1222 uint16_t devid, bool enable)
1223 {
1224 struct amdvi_dte* temp;
1225
1226 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
1227 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
1228
1229 temp = &amdvi_dte[devid];
1230
1231 #ifdef AMDVI_ATS_ENABLE
1232 /* If IOMMU and device support IOTLB, enable it. */
1233 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1234 temp->iotlb_enable = 1;
1235 #endif
1236
1237 /* Avoid duplicate I/O faults. */
1238 temp->sup_second_io_fault = 1;
1239 temp->sup_all_io_fault = amdvi_disable_io_fault;
1240
1241 temp->dt_valid = 1;
1242 temp->domain_id = domain->id;
1243
1244 if (enable) {
1245 if (domain->ptp) {
1246 temp->pt_base = vtophys(domain->ptp) >> 12;
1247 temp->pt_level = amdvi_ptp_level;
1248 }
1249 /*
1250 * XXX: Page table valid[TV] bit must be set even if host domain
1251 * page tables are not enabled.
1252 */
1253 temp->pt_valid = 1;
1254 temp->read_allow = 1;
1255 temp->write_allow = 1;
1256 }
1257 }
1258
1259 static void
amdvi_inv_device(struct amdvi_softc * softc,uint16_t devid)1260 amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid)
1261 {
1262 KASSERT(softc, ("softc is NULL"));
1263
1264 amdvi_cmd_inv_dte(softc, devid);
1265 #ifdef AMDVI_ATS_ENABLE
1266 if (amdvi_dev_support_iotlb(softc, devid))
1267 amdvi_cmd_inv_iotlb(softc, devid);
1268 #endif
1269 amdvi_wait(softc);
1270 }
1271
1272 static void
amdvi_add_device(void * arg,uint16_t devid)1273 amdvi_add_device(void *arg, uint16_t devid)
1274 {
1275 struct amdvi_domain *domain;
1276 struct amdvi_softc *softc;
1277
1278 domain = (struct amdvi_domain *)arg;
1279 KASSERT(domain != NULL, ("domain is NULL"));
1280 #ifdef AMDVI_DEBUG_CMD
1281 printf("Assigning device(%d.%d.%d) to domain:%d\n",
1282 RID2PCI_STR(devid), domain->id);
1283 #endif
1284 softc = amdvi_find_iommu(devid);
1285 if (softc == NULL)
1286 return;
1287 amdvi_set_dte(domain, softc, devid, true);
1288 amdvi_inv_device(softc, devid);
1289 }
1290
1291 static void
amdvi_remove_device(void * arg,uint16_t devid)1292 amdvi_remove_device(void *arg, uint16_t devid)
1293 {
1294 struct amdvi_domain *domain;
1295 struct amdvi_softc *softc;
1296
1297 domain = (struct amdvi_domain *)arg;
1298 #ifdef AMDVI_DEBUG_CMD
1299 printf("Remove device(0x%x) from domain:%d\n",
1300 devid, domain->id);
1301 #endif
1302 softc = amdvi_find_iommu(devid);
1303 if (softc == NULL)
1304 return;
1305 amdvi_set_dte(domain, softc, devid, false);
1306 amdvi_inv_device(softc, devid);
1307 }
1308
1309 static void
amdvi_enable(void)1310 amdvi_enable(void)
1311 {
1312 struct amdvi_ctrl *ctrl;
1313 struct amdvi_softc *softc;
1314 uint64_t val;
1315 int i;
1316
1317 for (i = 0; i < ivhd_count; i++) {
1318 softc = device_get_softc(ivhd_devs[i]);
1319 KASSERT(softc, ("softc is NULL\n"));
1320 ctrl = softc->ctrl;
1321 KASSERT(ctrl, ("ctrl is NULL\n"));
1322
1323 val = ( AMDVI_CTRL_EN |
1324 AMDVI_CTRL_CMD |
1325 AMDVI_CTRL_ELOG |
1326 AMDVI_CTRL_ELOGINT |
1327 AMDVI_CTRL_INV_TO_1S);
1328
1329 if (softc->ivhd_flag & IVHD_FLAG_COH)
1330 val |= AMDVI_CTRL_COH;
1331 if (softc->ivhd_flag & IVHD_FLAG_HTT)
1332 val |= AMDVI_CTRL_HTT;
1333 if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1334 val |= AMDVI_CTRL_RPPW;
1335 if (softc->ivhd_flag & IVHD_FLAG_PPW)
1336 val |= AMDVI_CTRL_PPW;
1337 if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1338 val |= AMDVI_CTRL_ISOC;
1339
1340 ctrl->control = val;
1341 }
1342 }
1343
1344 static void
amdvi_disable(void)1345 amdvi_disable(void)
1346 {
1347 struct amdvi_ctrl *ctrl;
1348 struct amdvi_softc *softc;
1349 int i;
1350
1351 for (i = 0; i < ivhd_count; i++) {
1352 softc = device_get_softc(ivhd_devs[i]);
1353 KASSERT(softc, ("softc is NULL\n"));
1354 ctrl = softc->ctrl;
1355 KASSERT(ctrl, ("ctrl is NULL\n"));
1356
1357 ctrl->control = 0;
1358 }
1359 }
1360
1361 static void
amdvi_invalidate_tlb(void * arg)1362 amdvi_invalidate_tlb(void *arg)
1363 {
1364 struct amdvi_domain *domain;
1365
1366 domain = (struct amdvi_domain *)arg;
1367 KASSERT(domain, ("domain is NULL"));
1368 amdvi_do_inv_domain(domain->id, false);
1369 }
1370
1371 const struct iommu_ops iommu_ops_amd = {
1372 .init = amdvi_init,
1373 .cleanup = amdvi_cleanup,
1374 .enable = amdvi_enable,
1375 .disable = amdvi_disable,
1376 .create_domain = amdvi_create_domain,
1377 .destroy_domain = amdvi_destroy_domain,
1378 .create_mapping = amdvi_create_mapping,
1379 .remove_mapping = amdvi_remove_mapping,
1380 .add_device = amdvi_add_device,
1381 .remove_device = amdvi_remove_device,
1382 .invalidate_tlb = amdvi_invalidate_tlb
1383 };
1384