1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sysctl.h>
48 #include <sys/systm.h>
49 #include <sys/taskqueue.h>
50 #include <sys/tree.h>
51
52 #include <vm/vm.h>
53 #include <vm/pmap.h>
54 #include <vm/vm_extern.h>
55
56 #include <sys/bus.h>
57 #include <machine/bus.h>
58 #include <sys/rman.h>
59 #include <machine/resource.h>
60 #include <machine/stdarg.h>
61
62 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
63 #include <machine/intr_machdep.h>
64 #endif
65
66 #include <sys/pciio.h>
67 #include <dev/pci/pcireg.h>
68 #include <dev/pci/pcivar.h>
69 #include <dev/pci/pci_private.h>
70
71 #ifdef PCI_IOV
72 #include <sys/nv.h>
73 #include <dev/pci/pci_iov_private.h>
74 #endif
75
76 #include <dev/usb/controller/xhcireg.h>
77 #include <dev/usb/controller/ehcireg.h>
78 #include <dev/usb/controller/ohcireg.h>
79 #include <dev/usb/controller/uhcireg.h>
80
81 #include <dev/iommu/iommu.h>
82
83 #include "pcib_if.h"
84 #include "pci_if.h"
85
86 #define PCIR_IS_BIOS(cfg, reg) \
87 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
88 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
89
90 static int pci_has_quirk(uint32_t devid, int quirk);
91 static pci_addr_t pci_mapbase(uint64_t mapreg);
92 static const char *pci_maptype(uint64_t mapreg);
93 static int pci_maprange(uint64_t mapreg);
94 static pci_addr_t pci_rombase(uint64_t mapreg);
95 static int pci_romsize(uint64_t testval);
96 static void pci_fixancient(pcicfgregs *cfg);
97 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
98
99 static int pci_porten(device_t dev);
100 static int pci_memen(device_t dev);
101 static void pci_assign_interrupt(device_t bus, device_t dev,
102 int force_route);
103 static int pci_add_map(device_t bus, device_t dev, int reg,
104 struct resource_list *rl, int force, int prefetch);
105 static int pci_probe(device_t dev);
106 static void pci_load_vendor_data(void);
107 static int pci_describe_parse_line(char **ptr, int *vendor,
108 int *device, char **desc);
109 static char *pci_describe_device(device_t dev);
110 static int pci_modevent(module_t mod, int what, void *arg);
111 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
112 pcicfgregs *cfg);
113 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
114 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
115 int reg, uint32_t *data);
116 #if 0
117 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
118 int reg, uint32_t data);
119 #endif
120 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
121 static void pci_mask_msix(device_t dev, u_int index);
122 static void pci_unmask_msix(device_t dev, u_int index);
123 static int pci_msi_blacklisted(void);
124 static int pci_msix_blacklisted(void);
125 static void pci_resume_msi(device_t dev);
126 static void pci_resume_msix(device_t dev);
127 static int pci_remap_intr_method(device_t bus, device_t dev,
128 u_int irq);
129 static void pci_hint_device_unit(device_t acdev, device_t child,
130 const char *name, int *unitp);
131 static int pci_reset_post(device_t dev, device_t child);
132 static int pci_reset_prepare(device_t dev, device_t child);
133 static int pci_reset_child(device_t dev, device_t child,
134 int flags);
135
136 static int pci_get_id_method(device_t dev, device_t child,
137 enum pci_id_type type, uintptr_t *rid);
138
139 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
140 int b, int s, int f, uint16_t vid, uint16_t did);
141
142 static device_method_t pci_methods[] = {
143 /* Device interface */
144 DEVMETHOD(device_probe, pci_probe),
145 DEVMETHOD(device_attach, pci_attach),
146 DEVMETHOD(device_detach, pci_detach),
147 DEVMETHOD(device_shutdown, bus_generic_shutdown),
148 DEVMETHOD(device_suspend, bus_generic_suspend),
149 DEVMETHOD(device_resume, pci_resume),
150
151 /* Bus interface */
152 DEVMETHOD(bus_print_child, pci_print_child),
153 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
154 DEVMETHOD(bus_read_ivar, pci_read_ivar),
155 DEVMETHOD(bus_write_ivar, pci_write_ivar),
156 DEVMETHOD(bus_driver_added, pci_driver_added),
157 DEVMETHOD(bus_setup_intr, pci_setup_intr),
158 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
159 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
160 DEVMETHOD(bus_reset_post, pci_reset_post),
161 DEVMETHOD(bus_reset_child, pci_reset_child),
162
163 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
164 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
165 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
166 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
167 DEVMETHOD(bus_delete_resource, pci_delete_resource),
168 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
169 DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
170 DEVMETHOD(bus_release_resource, pci_release_resource),
171 DEVMETHOD(bus_activate_resource, pci_activate_resource),
172 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
173 DEVMETHOD(bus_child_deleted, pci_child_deleted),
174 DEVMETHOD(bus_child_detached, pci_child_detached),
175 DEVMETHOD(bus_child_pnpinfo_str, pci_child_pnpinfo_str_method),
176 DEVMETHOD(bus_child_location_str, pci_child_location_str_method),
177 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
178 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
179 DEVMETHOD(bus_suspend_child, pci_suspend_child),
180 DEVMETHOD(bus_resume_child, pci_resume_child),
181 DEVMETHOD(bus_rescan, pci_rescan_method),
182
183 /* PCI interface */
184 DEVMETHOD(pci_read_config, pci_read_config_method),
185 DEVMETHOD(pci_write_config, pci_write_config_method),
186 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
187 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
188 DEVMETHOD(pci_enable_io, pci_enable_io_method),
189 DEVMETHOD(pci_disable_io, pci_disable_io_method),
190 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
191 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
192 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
193 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
194 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
195 DEVMETHOD(pci_find_cap, pci_find_cap_method),
196 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
197 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
198 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
199 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
200 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
201 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
202 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
203 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
204 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
205 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
206 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
207 DEVMETHOD(pci_release_msi, pci_release_msi_method),
208 DEVMETHOD(pci_msi_count, pci_msi_count_method),
209 DEVMETHOD(pci_msix_count, pci_msix_count_method),
210 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
211 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
212 DEVMETHOD(pci_get_id, pci_get_id_method),
213 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
214 DEVMETHOD(pci_child_added, pci_child_added_method),
215 #ifdef PCI_IOV
216 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
217 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
218 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
219 #endif
220
221 DEVMETHOD_END
222 };
223
224 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
225
226 static devclass_t pci_devclass;
227 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL,
228 BUS_PASS_BUS);
229 MODULE_VERSION(pci, 1);
230
231 static char *pci_vendordata;
232 static size_t pci_vendordata_size;
233
234 struct pci_quirk {
235 uint32_t devid; /* Vendor/device of the card */
236 int type;
237 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
238 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
239 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
240 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
241 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
242 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
243 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
244 int arg1;
245 int arg2;
246 };
247
248 static const struct pci_quirk pci_quirks[] = {
249 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
250 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
251 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
252 /* As does the Serverworks OSB4 (the SMBus mapping register) */
253 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
254
255 /*
256 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
257 * or the CMIC-SL (AKA ServerWorks GC_LE).
258 */
259 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
260 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
261
262 /*
263 * MSI doesn't work on earlier Intel chipsets including
264 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
265 */
266 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
267 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
268 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
269 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
270 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
271 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
273
274 /*
275 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
276 * bridge.
277 */
278 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
279
280 /*
281 * Some virtualization environments emulate an older chipset
282 * but support MSI just fine. QEMU uses the Intel 82440.
283 */
284 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
285
286 /*
287 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
288 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
289 * It prevents us from attaching hpet(4) when the bit is unset.
290 * Note this quirk only affects SB600 revision A13 and earlier.
291 * For SB600 A21 and later, firmware must set the bit to hide it.
292 * For SB700 and later, it is unused and hardcoded to zero.
293 */
294 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
295
296 /*
297 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
298 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
299 * of the command register is set.
300 */
301 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
302 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
303 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
304 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
305 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
306
307 /*
308 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
309 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
310 */
311 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
312 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
313 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
314 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
315 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
316 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
317
318 /*
319 * HPE Gen 10 VGA has a memory range that can't be allocated in the
320 * expected place.
321 */
322 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
323 { 0 }
324 };
325
326 /* map register information */
327 #define PCI_MAPMEM 0x01 /* memory map */
328 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
329 #define PCI_MAPPORT 0x04 /* port map */
330
331 struct devlist pci_devq;
332 uint32_t pci_generation;
333 uint32_t pci_numdevs = 0;
334 static int pcie_chipset, pcix_chipset;
335
336 /* sysctl vars */
337 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
338 "PCI bus tuning parameters");
339
340 static int pci_enable_io_modes = 1;
341 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
342 &pci_enable_io_modes, 1,
343 "Enable I/O and memory bits in the config register. Some BIOSes do not"
344 " enable these bits correctly. We'd like to do this all the time, but"
345 " there are some peripherals that this causes problems with.");
346
347 static int pci_do_realloc_bars = 1;
348 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
349 &pci_do_realloc_bars, 0,
350 "Attempt to allocate a new range for any BARs whose original "
351 "firmware-assigned ranges fail to allocate during the initial device scan.");
352
353 static int pci_do_power_nodriver = 0;
354 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
355 &pci_do_power_nodriver, 0,
356 "Place a function into D3 state when no driver attaches to it. 0 means"
357 " disable. 1 means conservatively place devices into D3 state. 2 means"
358 " aggressively place devices into D3 state. 3 means put absolutely"
359 " everything in D3 state.");
360
361 int pci_do_power_resume = 1;
362 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
363 &pci_do_power_resume, 1,
364 "Transition from D3 -> D0 on resume.");
365
366 int pci_do_power_suspend = 1;
367 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
368 &pci_do_power_suspend, 1,
369 "Transition from D0 -> D3 on suspend.");
370
371 static int pci_do_msi = 1;
372 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
373 "Enable support for MSI interrupts");
374
375 static int pci_do_msix = 1;
376 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
377 "Enable support for MSI-X interrupts");
378
379 static int pci_msix_rewrite_table = 0;
380 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
381 &pci_msix_rewrite_table, 0,
382 "Rewrite entire MSI-X table when updating MSI-X entries");
383
384 static int pci_honor_msi_blacklist = 1;
385 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
386 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
387
388 #if defined(__i386__) || defined(__amd64__)
389 static int pci_usb_takeover = 1;
390 #else
391 static int pci_usb_takeover = 0;
392 #endif
393 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
394 &pci_usb_takeover, 1,
395 "Enable early takeover of USB controllers. Disable this if you depend on"
396 " BIOS emulation of USB devices, that is you use USB devices (like"
397 " keyboard or mouse) but do not load USB drivers");
398
399 static int pci_clear_bars;
400 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
401 "Ignore firmware-assigned resources for BARs.");
402
403 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
404 static int pci_clear_buses;
405 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
406 "Ignore firmware-assigned bus numbers.");
407 #endif
408
409 static int pci_enable_ari = 1;
410 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
411 0, "Enable support for PCIe Alternative RID Interpretation");
412
413 int pci_enable_aspm = 1;
414 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
415 0, "Enable support for PCIe Active State Power Management");
416
417 static int pci_clear_aer_on_attach = 0;
418 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
419 &pci_clear_aer_on_attach, 0,
420 "Clear port and device AER state on driver attach");
421
422 static bool pci_enable_mps_tune = true;
423 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
424 &pci_enable_mps_tune, 1,
425 "Enable tuning of MPS(maximum payload size)." );
426
427 static int
pci_has_quirk(uint32_t devid,int quirk)428 pci_has_quirk(uint32_t devid, int quirk)
429 {
430 const struct pci_quirk *q;
431
432 for (q = &pci_quirks[0]; q->devid; q++) {
433 if (q->devid == devid && q->type == quirk)
434 return (1);
435 }
436 return (0);
437 }
438
439 /* Find a device_t by bus/slot/function in domain 0 */
440
441 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)442 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
443 {
444
445 return (pci_find_dbsf(0, bus, slot, func));
446 }
447
448 /* Find a device_t by domain/bus/slot/function */
449
450 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)451 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
452 {
453 struct pci_devinfo *dinfo = NULL;
454
455 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
456 if ((dinfo->cfg.domain == domain) &&
457 (dinfo->cfg.bus == bus) &&
458 (dinfo->cfg.slot == slot) &&
459 (dinfo->cfg.func == func)) {
460 break;
461 }
462 }
463
464 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
465 }
466
467 /* Find a device_t by vendor/device ID */
468
469 device_t
pci_find_device(uint16_t vendor,uint16_t device)470 pci_find_device(uint16_t vendor, uint16_t device)
471 {
472 struct pci_devinfo *dinfo;
473
474 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
475 if ((dinfo->cfg.vendor == vendor) &&
476 (dinfo->cfg.device == device)) {
477 return (dinfo->cfg.dev);
478 }
479 }
480
481 return (NULL);
482 }
483
484 device_t
pci_find_class(uint8_t class,uint8_t subclass)485 pci_find_class(uint8_t class, uint8_t subclass)
486 {
487 struct pci_devinfo *dinfo;
488
489 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
490 if (dinfo->cfg.baseclass == class &&
491 dinfo->cfg.subclass == subclass) {
492 return (dinfo->cfg.dev);
493 }
494 }
495
496 return (NULL);
497 }
498
499 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)500 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
501 {
502 struct pci_devinfo *dinfo;
503 bool found = false;
504
505 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
506 if (from != NULL && found == false) {
507 if (from != dinfo->cfg.dev)
508 continue;
509 found = true;
510 continue;
511 }
512 if (dinfo->cfg.baseclass == class &&
513 dinfo->cfg.subclass == subclass) {
514 return (dinfo->cfg.dev);
515 }
516 }
517
518 return (NULL);
519 }
520
521 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)522 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
523 {
524 va_list ap;
525 int retval;
526
527 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
528 cfg->func);
529 va_start(ap, fmt);
530 retval += vprintf(fmt, ap);
531 va_end(ap);
532 return (retval);
533 }
534
535 /* return base address of memory or port map */
536
537 static pci_addr_t
pci_mapbase(uint64_t mapreg)538 pci_mapbase(uint64_t mapreg)
539 {
540
541 if (PCI_BAR_MEM(mapreg))
542 return (mapreg & PCIM_BAR_MEM_BASE);
543 else
544 return (mapreg & PCIM_BAR_IO_BASE);
545 }
546
547 /* return map type of memory or port map */
548
549 static const char *
pci_maptype(uint64_t mapreg)550 pci_maptype(uint64_t mapreg)
551 {
552
553 if (PCI_BAR_IO(mapreg))
554 return ("I/O Port");
555 if (mapreg & PCIM_BAR_MEM_PREFETCH)
556 return ("Prefetchable Memory");
557 return ("Memory");
558 }
559
560 /* return log2 of map size decoded for memory or port map */
561
562 int
pci_mapsize(uint64_t testval)563 pci_mapsize(uint64_t testval)
564 {
565 int ln2size;
566
567 testval = pci_mapbase(testval);
568 ln2size = 0;
569 if (testval != 0) {
570 while ((testval & 1) == 0)
571 {
572 ln2size++;
573 testval >>= 1;
574 }
575 }
576 return (ln2size);
577 }
578
579 /* return base address of device ROM */
580
581 static pci_addr_t
pci_rombase(uint64_t mapreg)582 pci_rombase(uint64_t mapreg)
583 {
584
585 return (mapreg & PCIM_BIOS_ADDR_MASK);
586 }
587
588 /* return log2 of map size decided for device ROM */
589
590 static int
pci_romsize(uint64_t testval)591 pci_romsize(uint64_t testval)
592 {
593 int ln2size;
594
595 testval = pci_rombase(testval);
596 ln2size = 0;
597 if (testval != 0) {
598 while ((testval & 1) == 0)
599 {
600 ln2size++;
601 testval >>= 1;
602 }
603 }
604 return (ln2size);
605 }
606
607 /* return log2 of address range supported by map register */
608
609 static int
pci_maprange(uint64_t mapreg)610 pci_maprange(uint64_t mapreg)
611 {
612 int ln2range = 0;
613
614 if (PCI_BAR_IO(mapreg))
615 ln2range = 32;
616 else
617 switch (mapreg & PCIM_BAR_MEM_TYPE) {
618 case PCIM_BAR_MEM_32:
619 ln2range = 32;
620 break;
621 case PCIM_BAR_MEM_1MB:
622 ln2range = 20;
623 break;
624 case PCIM_BAR_MEM_64:
625 ln2range = 64;
626 break;
627 }
628 return (ln2range);
629 }
630
631 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
632
633 static void
pci_fixancient(pcicfgregs * cfg)634 pci_fixancient(pcicfgregs *cfg)
635 {
636 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
637 return;
638
639 /* PCI to PCI bridges use header type 1 */
640 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
641 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
642 }
643
644 /* extract header type specific config data */
645
646 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)647 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
648 {
649 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
650 switch (cfg->hdrtype & PCIM_HDRTYPE) {
651 case PCIM_HDRTYPE_NORMAL:
652 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
653 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
654 cfg->mingnt = REG(PCIR_MINGNT, 1);
655 cfg->maxlat = REG(PCIR_MAXLAT, 1);
656 cfg->nummaps = PCI_MAXMAPS_0;
657 break;
658 case PCIM_HDRTYPE_BRIDGE:
659 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
660 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
661 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
662 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
663 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
664 cfg->nummaps = PCI_MAXMAPS_1;
665 break;
666 case PCIM_HDRTYPE_CARDBUS:
667 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
668 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
669 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
670 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
671 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
672 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
673 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
674 cfg->nummaps = PCI_MAXMAPS_2;
675 break;
676 }
677 #undef REG
678 }
679
680 /* read configuration header into pcicfgregs structure */
681 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)682 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
683 {
684 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
685 uint16_t vid, did;
686
687 vid = REG(PCIR_VENDOR, 2);
688 if (vid == PCIV_INVALID)
689 return (NULL);
690
691 did = REG(PCIR_DEVICE, 2);
692
693 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
694 }
695
696 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)697 pci_alloc_devinfo_method(device_t dev)
698 {
699
700 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
701 M_WAITOK | M_ZERO));
702 }
703
704 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)705 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
706 uint16_t vid, uint16_t did)
707 {
708 struct pci_devinfo *devlist_entry;
709 pcicfgregs *cfg;
710
711 devlist_entry = PCI_ALLOC_DEVINFO(bus);
712
713 cfg = &devlist_entry->cfg;
714
715 cfg->domain = d;
716 cfg->bus = b;
717 cfg->slot = s;
718 cfg->func = f;
719 cfg->vendor = vid;
720 cfg->device = did;
721 cfg->cmdreg = REG(PCIR_COMMAND, 2);
722 cfg->statreg = REG(PCIR_STATUS, 2);
723 cfg->baseclass = REG(PCIR_CLASS, 1);
724 cfg->subclass = REG(PCIR_SUBCLASS, 1);
725 cfg->progif = REG(PCIR_PROGIF, 1);
726 cfg->revid = REG(PCIR_REVID, 1);
727 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
728 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
729 cfg->lattimer = REG(PCIR_LATTIMER, 1);
730 cfg->intpin = REG(PCIR_INTPIN, 1);
731 cfg->intline = REG(PCIR_INTLINE, 1);
732
733 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
734 cfg->hdrtype &= ~PCIM_MFDEV;
735 STAILQ_INIT(&cfg->maps);
736
737 cfg->iov = NULL;
738
739 pci_fixancient(cfg);
740 pci_hdrtypedata(pcib, b, s, f, cfg);
741
742 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
743 pci_read_cap(pcib, cfg);
744
745 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
746
747 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
748 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
749 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
750 devlist_entry->conf.pc_sel.pc_func = cfg->func;
751 devlist_entry->conf.pc_hdr = cfg->hdrtype;
752
753 devlist_entry->conf.pc_subvendor = cfg->subvendor;
754 devlist_entry->conf.pc_subdevice = cfg->subdevice;
755 devlist_entry->conf.pc_vendor = cfg->vendor;
756 devlist_entry->conf.pc_device = cfg->device;
757
758 devlist_entry->conf.pc_class = cfg->baseclass;
759 devlist_entry->conf.pc_subclass = cfg->subclass;
760 devlist_entry->conf.pc_progif = cfg->progif;
761 devlist_entry->conf.pc_revid = cfg->revid;
762
763 pci_numdevs++;
764 pci_generation++;
765
766 return (devlist_entry);
767 }
768 #undef REG
769
770 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)771 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
772 {
773 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
774 cfg->ea.ea_location + (n), w)
775 int num_ent;
776 int ptr;
777 int a, b;
778 uint32_t val;
779 int ent_size;
780 uint32_t dw[4];
781 uint64_t base, max_offset;
782 struct pci_ea_entry *eae;
783
784 if (cfg->ea.ea_location == 0)
785 return;
786
787 STAILQ_INIT(&cfg->ea.ea_entries);
788
789 /* Determine the number of entries */
790 num_ent = REG(PCIR_EA_NUM_ENT, 2);
791 num_ent &= PCIM_EA_NUM_ENT_MASK;
792
793 /* Find the first entry to care of */
794 ptr = PCIR_EA_FIRST_ENT;
795
796 /* Skip DWORD 2 for type 1 functions */
797 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
798 ptr += 4;
799
800 for (a = 0; a < num_ent; a++) {
801 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
802 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
803
804 /* Read a number of dwords in the entry */
805 val = REG(ptr, 4);
806 ptr += 4;
807 ent_size = (val & PCIM_EA_ES);
808
809 for (b = 0; b < ent_size; b++) {
810 dw[b] = REG(ptr, 4);
811 ptr += 4;
812 }
813
814 eae->eae_flags = val;
815 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
816
817 base = dw[0] & PCIM_EA_FIELD_MASK;
818 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
819 b = 2;
820 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
821 base |= (uint64_t)dw[b] << 32UL;
822 b++;
823 }
824 if (((dw[1] & PCIM_EA_IS_64) != 0)
825 && (b < ent_size)) {
826 max_offset |= (uint64_t)dw[b] << 32UL;
827 b++;
828 }
829
830 eae->eae_base = base;
831 eae->eae_max_offset = max_offset;
832
833 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
834
835 if (bootverbose) {
836 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
837 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
838 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
839 }
840 }
841 }
842 #undef REG
843
844 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)845 pci_read_cap(device_t pcib, pcicfgregs *cfg)
846 {
847 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
848 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
849 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
850 uint64_t addr;
851 #endif
852 uint32_t val;
853 int ptr, nextptr, ptrptr;
854
855 switch (cfg->hdrtype & PCIM_HDRTYPE) {
856 case PCIM_HDRTYPE_NORMAL:
857 case PCIM_HDRTYPE_BRIDGE:
858 ptrptr = PCIR_CAP_PTR;
859 break;
860 case PCIM_HDRTYPE_CARDBUS:
861 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
862 break;
863 default:
864 return; /* no extended capabilities support */
865 }
866 nextptr = REG(ptrptr, 1); /* sanity check? */
867
868 /*
869 * Read capability entries.
870 */
871 while (nextptr != 0) {
872 /* Sanity check */
873 if (nextptr > 255) {
874 printf("illegal PCI extended capability offset %d\n",
875 nextptr);
876 return;
877 }
878 /* Find the next entry */
879 ptr = nextptr;
880 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
881
882 /* Process this entry */
883 switch (REG(ptr + PCICAP_ID, 1)) {
884 case PCIY_PMG: /* PCI power management */
885 if (cfg->pp.pp_cap == 0) {
886 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
887 cfg->pp.pp_status = ptr + PCIR_POWER_STATUS;
888 cfg->pp.pp_bse = ptr + PCIR_POWER_BSE;
889 if ((nextptr - ptr) > PCIR_POWER_DATA)
890 cfg->pp.pp_data = ptr + PCIR_POWER_DATA;
891 }
892 break;
893 case PCIY_HT: /* HyperTransport */
894 /* Determine HT-specific capability type. */
895 val = REG(ptr + PCIR_HT_COMMAND, 2);
896
897 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
898 cfg->ht.ht_slave = ptr;
899
900 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
901 switch (val & PCIM_HTCMD_CAP_MASK) {
902 case PCIM_HTCAP_MSI_MAPPING:
903 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
904 /* Sanity check the mapping window. */
905 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
906 4);
907 addr <<= 32;
908 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
909 4);
910 if (addr != MSI_INTEL_ADDR_BASE)
911 device_printf(pcib,
912 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
913 cfg->domain, cfg->bus,
914 cfg->slot, cfg->func,
915 (long long)addr);
916 } else
917 addr = MSI_INTEL_ADDR_BASE;
918
919 cfg->ht.ht_msimap = ptr;
920 cfg->ht.ht_msictrl = val;
921 cfg->ht.ht_msiaddr = addr;
922 break;
923 }
924 #endif
925 break;
926 case PCIY_MSI: /* PCI MSI */
927 cfg->msi.msi_location = ptr;
928 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
929 cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl &
930 PCIM_MSICTRL_MMC_MASK)>>1);
931 break;
932 case PCIY_MSIX: /* PCI MSI-X */
933 cfg->msix.msix_location = ptr;
934 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
935 cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl &
936 PCIM_MSIXCTRL_TABLE_SIZE) + 1;
937 val = REG(ptr + PCIR_MSIX_TABLE, 4);
938 cfg->msix.msix_table_bar = PCIR_BAR(val &
939 PCIM_MSIX_BIR_MASK);
940 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
941 val = REG(ptr + PCIR_MSIX_PBA, 4);
942 cfg->msix.msix_pba_bar = PCIR_BAR(val &
943 PCIM_MSIX_BIR_MASK);
944 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
945 break;
946 case PCIY_VPD: /* PCI Vital Product Data */
947 cfg->vpd.vpd_reg = ptr;
948 break;
949 case PCIY_SUBVENDOR:
950 /* Should always be true. */
951 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
952 PCIM_HDRTYPE_BRIDGE) {
953 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
954 cfg->subvendor = val & 0xffff;
955 cfg->subdevice = val >> 16;
956 }
957 break;
958 case PCIY_PCIX: /* PCI-X */
959 /*
960 * Assume we have a PCI-X chipset if we have
961 * at least one PCI-PCI bridge with a PCI-X
962 * capability. Note that some systems with
963 * PCI-express or HT chipsets might match on
964 * this check as well.
965 */
966 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
967 PCIM_HDRTYPE_BRIDGE)
968 pcix_chipset = 1;
969 cfg->pcix.pcix_location = ptr;
970 break;
971 case PCIY_EXPRESS: /* PCI-express */
972 /*
973 * Assume we have a PCI-express chipset if we have
974 * at least one PCI-express device.
975 */
976 pcie_chipset = 1;
977 cfg->pcie.pcie_location = ptr;
978 val = REG(ptr + PCIER_FLAGS, 2);
979 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
980 break;
981 case PCIY_EA: /* Enhanced Allocation */
982 cfg->ea.ea_location = ptr;
983 pci_ea_fill_info(pcib, cfg);
984 break;
985 default:
986 break;
987 }
988 }
989
990 #if defined(__powerpc__)
991 /*
992 * Enable the MSI mapping window for all HyperTransport
993 * slaves. PCI-PCI bridges have their windows enabled via
994 * PCIB_MAP_MSI().
995 */
996 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
997 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
998 device_printf(pcib,
999 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
1000 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1001 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1002 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1003 2);
1004 }
1005 #endif
1006 /* REG and WREG use carry through to next functions */
1007 }
1008
1009 /*
1010 * PCI Vital Product Data
1011 */
1012
1013 #define PCI_VPD_TIMEOUT 1000000
1014
1015 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1016 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1017 {
1018 int count = PCI_VPD_TIMEOUT;
1019
1020 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1021
1022 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1023
1024 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1025 if (--count < 0)
1026 return (ENXIO);
1027 DELAY(1); /* limit looping */
1028 }
1029 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1030
1031 return (0);
1032 }
1033
1034 #if 0
1035 static int
1036 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1037 {
1038 int count = PCI_VPD_TIMEOUT;
1039
1040 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1041
1042 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1043 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1044 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1045 if (--count < 0)
1046 return (ENXIO);
1047 DELAY(1); /* limit looping */
1048 }
1049
1050 return (0);
1051 }
1052 #endif
1053
1054 #undef PCI_VPD_TIMEOUT
1055
1056 struct vpd_readstate {
1057 device_t pcib;
1058 pcicfgregs *cfg;
1059 uint32_t val;
1060 int bytesinval;
1061 int off;
1062 uint8_t cksum;
1063 };
1064
1065 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1066 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1067 {
1068 uint32_t reg;
1069 uint8_t byte;
1070
1071 if (vrs->bytesinval == 0) {
1072 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1073 return (ENXIO);
1074 vrs->val = le32toh(reg);
1075 vrs->off += 4;
1076 byte = vrs->val & 0xff;
1077 vrs->bytesinval = 3;
1078 } else {
1079 vrs->val = vrs->val >> 8;
1080 byte = vrs->val & 0xff;
1081 vrs->bytesinval--;
1082 }
1083
1084 vrs->cksum += byte;
1085 *data = byte;
1086 return (0);
1087 }
1088
1089 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1090 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1091 {
1092 struct vpd_readstate vrs;
1093 int state;
1094 int name;
1095 int remain;
1096 int i;
1097 int alloc, off; /* alloc/off for RO/W arrays */
1098 int cksumvalid;
1099 int dflen;
1100 int firstrecord;
1101 uint8_t byte;
1102 uint8_t byte2;
1103
1104 /* init vpd reader */
1105 vrs.bytesinval = 0;
1106 vrs.off = 0;
1107 vrs.pcib = pcib;
1108 vrs.cfg = cfg;
1109 vrs.cksum = 0;
1110
1111 state = 0;
1112 name = remain = i = 0; /* shut up stupid gcc */
1113 alloc = off = 0; /* shut up stupid gcc */
1114 dflen = 0; /* shut up stupid gcc */
1115 cksumvalid = -1;
1116 firstrecord = 1;
1117 while (state >= 0) {
1118 if (vpd_nextbyte(&vrs, &byte)) {
1119 pci_printf(cfg, "VPD read timed out\n");
1120 state = -2;
1121 break;
1122 }
1123 #if 0
1124 pci_printf(cfg, "vpd: val: %#x, off: %d, bytesinval: %d, byte: "
1125 "%#hhx, state: %d, remain: %d, name: %#x, i: %d\n", vrs.val,
1126 vrs.off, vrs.bytesinval, byte, state, remain, name, i);
1127 #endif
1128 switch (state) {
1129 case 0: /* item name */
1130 if (byte & 0x80) {
1131 if (vpd_nextbyte(&vrs, &byte2)) {
1132 state = -2;
1133 break;
1134 }
1135 remain = byte2;
1136 if (vpd_nextbyte(&vrs, &byte2)) {
1137 state = -2;
1138 break;
1139 }
1140 remain |= byte2 << 8;
1141 name = byte & 0x7f;
1142 } else {
1143 remain = byte & 0x7;
1144 name = (byte >> 3) & 0xf;
1145 }
1146 if (firstrecord) {
1147 if (name != 0x2) {
1148 pci_printf(cfg, "VPD data does not " \
1149 "start with ident (%#x)\n", name);
1150 state = -2;
1151 break;
1152 }
1153 firstrecord = 0;
1154 }
1155 if (vrs.off + remain - vrs.bytesinval > 0x8000) {
1156 pci_printf(cfg,
1157 "VPD data overflow, remain %#x\n", remain);
1158 state = -1;
1159 break;
1160 }
1161 switch (name) {
1162 case 0x2: /* String */
1163 if (cfg->vpd.vpd_ident != NULL) {
1164 pci_printf(cfg,
1165 "duplicate VPD ident record\n");
1166 state = -2;
1167 break;
1168 }
1169 if (remain > 255) {
1170 pci_printf(cfg,
1171 "VPD ident length %d exceeds 255\n",
1172 remain);
1173 state = -2;
1174 break;
1175 }
1176 cfg->vpd.vpd_ident = malloc(remain + 1,
1177 M_DEVBUF, M_WAITOK);
1178 i = 0;
1179 state = 1;
1180 break;
1181 case 0xf: /* End */
1182 state = -1;
1183 break;
1184 case 0x10: /* VPD-R */
1185 alloc = 8;
1186 off = 0;
1187 cfg->vpd.vpd_ros = malloc(alloc *
1188 sizeof(*cfg->vpd.vpd_ros), M_DEVBUF,
1189 M_WAITOK | M_ZERO);
1190 state = 2;
1191 break;
1192 case 0x11: /* VPD-W */
1193 alloc = 8;
1194 off = 0;
1195 cfg->vpd.vpd_w = malloc(alloc *
1196 sizeof(*cfg->vpd.vpd_w), M_DEVBUF,
1197 M_WAITOK | M_ZERO);
1198 state = 5;
1199 break;
1200 default: /* Invalid data, abort */
1201 pci_printf(cfg, "invalid VPD name: %#x\n", name);
1202 state = -2;
1203 break;
1204 }
1205 break;
1206
1207 case 1: /* Identifier String */
1208 cfg->vpd.vpd_ident[i++] = byte;
1209 remain--;
1210 if (remain == 0) {
1211 cfg->vpd.vpd_ident[i] = '\0';
1212 state = 0;
1213 }
1214 break;
1215
1216 case 2: /* VPD-R Keyword Header */
1217 if (off == alloc) {
1218 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
1219 (alloc *= 2) * sizeof(*cfg->vpd.vpd_ros),
1220 M_DEVBUF, M_WAITOK | M_ZERO);
1221 }
1222 cfg->vpd.vpd_ros[off].keyword[0] = byte;
1223 if (vpd_nextbyte(&vrs, &byte2)) {
1224 state = -2;
1225 break;
1226 }
1227 cfg->vpd.vpd_ros[off].keyword[1] = byte2;
1228 if (vpd_nextbyte(&vrs, &byte2)) {
1229 state = -2;
1230 break;
1231 }
1232 cfg->vpd.vpd_ros[off].len = dflen = byte2;
1233 if (dflen == 0 &&
1234 strncmp(cfg->vpd.vpd_ros[off].keyword, "RV",
1235 2) == 0) {
1236 /*
1237 * if this happens, we can't trust the rest
1238 * of the VPD.
1239 */
1240 pci_printf(cfg, "invalid VPD RV record");
1241 cksumvalid = 0;
1242 state = -1;
1243 break;
1244 } else if (dflen == 0) {
1245 cfg->vpd.vpd_ros[off].value = malloc(1 *
1246 sizeof(*cfg->vpd.vpd_ros[off].value),
1247 M_DEVBUF, M_WAITOK);
1248 cfg->vpd.vpd_ros[off].value[0] = '\x00';
1249 } else
1250 cfg->vpd.vpd_ros[off].value = malloc(
1251 (dflen + 1) *
1252 sizeof(*cfg->vpd.vpd_ros[off].value),
1253 M_DEVBUF, M_WAITOK);
1254 remain -= 3;
1255 i = 0;
1256 /* keep in sync w/ state 3's transitions */
1257 if (dflen == 0 && remain == 0)
1258 state = 0;
1259 else if (dflen == 0)
1260 state = 2;
1261 else
1262 state = 3;
1263 break;
1264
1265 case 3: /* VPD-R Keyword Value */
1266 cfg->vpd.vpd_ros[off].value[i++] = byte;
1267 if (strncmp(cfg->vpd.vpd_ros[off].keyword,
1268 "RV", 2) == 0 && cksumvalid == -1) {
1269 if (vrs.cksum == 0)
1270 cksumvalid = 1;
1271 else {
1272 if (bootverbose)
1273 pci_printf(cfg,
1274 "bad VPD cksum, remain %hhu\n",
1275 vrs.cksum);
1276 cksumvalid = 0;
1277 state = -1;
1278 break;
1279 }
1280 }
1281 dflen--;
1282 remain--;
1283 /* keep in sync w/ state 2's transitions */
1284 if (dflen == 0)
1285 cfg->vpd.vpd_ros[off++].value[i++] = '\0';
1286 if (dflen == 0 && remain == 0) {
1287 cfg->vpd.vpd_rocnt = off;
1288 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
1289 off * sizeof(*cfg->vpd.vpd_ros),
1290 M_DEVBUF, M_WAITOK | M_ZERO);
1291 state = 0;
1292 } else if (dflen == 0)
1293 state = 2;
1294 break;
1295
1296 case 4:
1297 remain--;
1298 if (remain == 0)
1299 state = 0;
1300 break;
1301
1302 case 5: /* VPD-W Keyword Header */
1303 if (off == alloc) {
1304 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
1305 (alloc *= 2) * sizeof(*cfg->vpd.vpd_w),
1306 M_DEVBUF, M_WAITOK | M_ZERO);
1307 }
1308 cfg->vpd.vpd_w[off].keyword[0] = byte;
1309 if (vpd_nextbyte(&vrs, &byte2)) {
1310 state = -2;
1311 break;
1312 }
1313 cfg->vpd.vpd_w[off].keyword[1] = byte2;
1314 if (vpd_nextbyte(&vrs, &byte2)) {
1315 state = -2;
1316 break;
1317 }
1318 cfg->vpd.vpd_w[off].len = dflen = byte2;
1319 cfg->vpd.vpd_w[off].start = vrs.off - vrs.bytesinval;
1320 cfg->vpd.vpd_w[off].value = malloc((dflen + 1) *
1321 sizeof(*cfg->vpd.vpd_w[off].value),
1322 M_DEVBUF, M_WAITOK);
1323 remain -= 3;
1324 i = 0;
1325 /* keep in sync w/ state 6's transitions */
1326 if (dflen == 0 && remain == 0)
1327 state = 0;
1328 else if (dflen == 0)
1329 state = 5;
1330 else
1331 state = 6;
1332 break;
1333
1334 case 6: /* VPD-W Keyword Value */
1335 cfg->vpd.vpd_w[off].value[i++] = byte;
1336 dflen--;
1337 remain--;
1338 /* keep in sync w/ state 5's transitions */
1339 if (dflen == 0)
1340 cfg->vpd.vpd_w[off++].value[i++] = '\0';
1341 if (dflen == 0 && remain == 0) {
1342 cfg->vpd.vpd_wcnt = off;
1343 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
1344 off * sizeof(*cfg->vpd.vpd_w),
1345 M_DEVBUF, M_WAITOK | M_ZERO);
1346 state = 0;
1347 } else if (dflen == 0)
1348 state = 5;
1349 break;
1350
1351 default:
1352 pci_printf(cfg, "invalid state: %d\n", state);
1353 state = -1;
1354 break;
1355 }
1356
1357 if (cfg->vpd.vpd_ident == NULL || cfg->vpd.vpd_ident[0] == '\0') {
1358 pci_printf(cfg, "no valid vpd ident found\n");
1359 state = -2;
1360 }
1361 }
1362
1363 if (cksumvalid <= 0 || state < -1) {
1364 /* read-only data bad, clean up */
1365 if (cfg->vpd.vpd_ros != NULL) {
1366 for (off = 0; cfg->vpd.vpd_ros[off].value; off++)
1367 free(cfg->vpd.vpd_ros[off].value, M_DEVBUF);
1368 free(cfg->vpd.vpd_ros, M_DEVBUF);
1369 cfg->vpd.vpd_ros = NULL;
1370 }
1371 }
1372 if (state < -1) {
1373 /* I/O error, clean up */
1374 pci_printf(cfg, "failed to read VPD data.\n");
1375 if (cfg->vpd.vpd_ident != NULL) {
1376 free(cfg->vpd.vpd_ident, M_DEVBUF);
1377 cfg->vpd.vpd_ident = NULL;
1378 }
1379 if (cfg->vpd.vpd_w != NULL) {
1380 for (off = 0; cfg->vpd.vpd_w[off].value; off++)
1381 free(cfg->vpd.vpd_w[off].value, M_DEVBUF);
1382 free(cfg->vpd.vpd_w, M_DEVBUF);
1383 cfg->vpd.vpd_w = NULL;
1384 }
1385 }
1386 cfg->vpd.vpd_cached = 1;
1387 #undef REG
1388 #undef WREG
1389 }
1390
1391 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1392 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1393 {
1394 struct pci_devinfo *dinfo = device_get_ivars(child);
1395 pcicfgregs *cfg = &dinfo->cfg;
1396
1397 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1398 pci_read_vpd(device_get_parent(dev), cfg);
1399
1400 *identptr = cfg->vpd.vpd_ident;
1401
1402 if (*identptr == NULL)
1403 return (ENXIO);
1404
1405 return (0);
1406 }
1407
1408 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1409 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1410 const char **vptr)
1411 {
1412 struct pci_devinfo *dinfo = device_get_ivars(child);
1413 pcicfgregs *cfg = &dinfo->cfg;
1414 int i;
1415
1416 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1417 pci_read_vpd(device_get_parent(dev), cfg);
1418
1419 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1420 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1421 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1422 *vptr = cfg->vpd.vpd_ros[i].value;
1423 return (0);
1424 }
1425
1426 *vptr = NULL;
1427 return (ENXIO);
1428 }
1429
1430 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1431 pci_fetch_vpd_list(device_t dev)
1432 {
1433 struct pci_devinfo *dinfo = device_get_ivars(dev);
1434 pcicfgregs *cfg = &dinfo->cfg;
1435
1436 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1437 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1438 return (&cfg->vpd);
1439 }
1440
1441 /*
1442 * Find the requested HyperTransport capability and return the offset
1443 * in configuration space via the pointer provided. The function
1444 * returns 0 on success and an error code otherwise.
1445 */
1446 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1447 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1448 {
1449 int ptr, error;
1450 uint16_t val;
1451
1452 error = pci_find_cap(child, PCIY_HT, &ptr);
1453 if (error)
1454 return (error);
1455
1456 /*
1457 * Traverse the capabilities list checking each HT capability
1458 * to see if it matches the requested HT capability.
1459 */
1460 for (;;) {
1461 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1462 if (capability == PCIM_HTCAP_SLAVE ||
1463 capability == PCIM_HTCAP_HOST)
1464 val &= 0xe000;
1465 else
1466 val &= PCIM_HTCMD_CAP_MASK;
1467 if (val == capability) {
1468 if (capreg != NULL)
1469 *capreg = ptr;
1470 return (0);
1471 }
1472
1473 /* Skip to the next HT capability. */
1474 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1475 break;
1476 }
1477
1478 return (ENOENT);
1479 }
1480
1481 /*
1482 * Find the next requested HyperTransport capability after start and return
1483 * the offset in configuration space via the pointer provided. The function
1484 * returns 0 on success and an error code otherwise.
1485 */
1486 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1487 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1488 int start, int *capreg)
1489 {
1490 int ptr;
1491 uint16_t val;
1492
1493 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1494 ("start capability is not HyperTransport capability"));
1495 ptr = start;
1496
1497 /*
1498 * Traverse the capabilities list checking each HT capability
1499 * to see if it matches the requested HT capability.
1500 */
1501 for (;;) {
1502 /* Skip to the next HT capability. */
1503 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1504 break;
1505
1506 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1507 if (capability == PCIM_HTCAP_SLAVE ||
1508 capability == PCIM_HTCAP_HOST)
1509 val &= 0xe000;
1510 else
1511 val &= PCIM_HTCMD_CAP_MASK;
1512 if (val == capability) {
1513 if (capreg != NULL)
1514 *capreg = ptr;
1515 return (0);
1516 }
1517 }
1518
1519 return (ENOENT);
1520 }
1521
1522 /*
1523 * Find the requested capability and return the offset in
1524 * configuration space via the pointer provided. The function returns
1525 * 0 on success and an error code otherwise.
1526 */
1527 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1528 pci_find_cap_method(device_t dev, device_t child, int capability,
1529 int *capreg)
1530 {
1531 struct pci_devinfo *dinfo = device_get_ivars(child);
1532 pcicfgregs *cfg = &dinfo->cfg;
1533 uint32_t status;
1534 uint8_t ptr;
1535
1536 /*
1537 * Check the CAP_LIST bit of the PCI status register first.
1538 */
1539 status = pci_read_config(child, PCIR_STATUS, 2);
1540 if (!(status & PCIM_STATUS_CAPPRESENT))
1541 return (ENXIO);
1542
1543 /*
1544 * Determine the start pointer of the capabilities list.
1545 */
1546 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1547 case PCIM_HDRTYPE_NORMAL:
1548 case PCIM_HDRTYPE_BRIDGE:
1549 ptr = PCIR_CAP_PTR;
1550 break;
1551 case PCIM_HDRTYPE_CARDBUS:
1552 ptr = PCIR_CAP_PTR_2;
1553 break;
1554 default:
1555 /* XXX: panic? */
1556 return (ENXIO); /* no extended capabilities support */
1557 }
1558 ptr = pci_read_config(child, ptr, 1);
1559
1560 /*
1561 * Traverse the capabilities list.
1562 */
1563 while (ptr != 0) {
1564 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1565 if (capreg != NULL)
1566 *capreg = ptr;
1567 return (0);
1568 }
1569 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1570 }
1571
1572 return (ENOENT);
1573 }
1574
1575 /*
1576 * Find the next requested capability after start and return the offset in
1577 * configuration space via the pointer provided. The function returns
1578 * 0 on success and an error code otherwise.
1579 */
1580 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1581 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1582 int start, int *capreg)
1583 {
1584 uint8_t ptr;
1585
1586 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1587 ("start capability is not expected capability"));
1588
1589 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1590 while (ptr != 0) {
1591 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1592 if (capreg != NULL)
1593 *capreg = ptr;
1594 return (0);
1595 }
1596 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1597 }
1598
1599 return (ENOENT);
1600 }
1601
1602 /*
1603 * Find the requested extended capability and return the offset in
1604 * configuration space via the pointer provided. The function returns
1605 * 0 on success and an error code otherwise.
1606 */
1607 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1608 pci_find_extcap_method(device_t dev, device_t child, int capability,
1609 int *capreg)
1610 {
1611 struct pci_devinfo *dinfo = device_get_ivars(child);
1612 pcicfgregs *cfg = &dinfo->cfg;
1613 uint32_t ecap;
1614 uint16_t ptr;
1615
1616 /* Only supported for PCI-express devices. */
1617 if (cfg->pcie.pcie_location == 0)
1618 return (ENXIO);
1619
1620 ptr = PCIR_EXTCAP;
1621 ecap = pci_read_config(child, ptr, 4);
1622 if (ecap == 0xffffffff || ecap == 0)
1623 return (ENOENT);
1624 for (;;) {
1625 if (PCI_EXTCAP_ID(ecap) == capability) {
1626 if (capreg != NULL)
1627 *capreg = ptr;
1628 return (0);
1629 }
1630 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1631 if (ptr == 0)
1632 break;
1633 ecap = pci_read_config(child, ptr, 4);
1634 }
1635
1636 return (ENOENT);
1637 }
1638
1639 /*
1640 * Find the next requested extended capability after start and return the
1641 * offset in configuration space via the pointer provided. The function
1642 * returns 0 on success and an error code otherwise.
1643 */
1644 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1645 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1646 int start, int *capreg)
1647 {
1648 struct pci_devinfo *dinfo = device_get_ivars(child);
1649 pcicfgregs *cfg = &dinfo->cfg;
1650 uint32_t ecap;
1651 uint16_t ptr;
1652
1653 /* Only supported for PCI-express devices. */
1654 if (cfg->pcie.pcie_location == 0)
1655 return (ENXIO);
1656
1657 ecap = pci_read_config(child, start, 4);
1658 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1659 ("start extended capability is not expected capability"));
1660 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1661 while (ptr != 0) {
1662 ecap = pci_read_config(child, ptr, 4);
1663 if (PCI_EXTCAP_ID(ecap) == capability) {
1664 if (capreg != NULL)
1665 *capreg = ptr;
1666 return (0);
1667 }
1668 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1669 }
1670
1671 return (ENOENT);
1672 }
1673
1674 /*
1675 * Support for MSI-X message interrupts.
1676 */
1677 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1678 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1679 {
1680 struct pci_devinfo *dinfo = device_get_ivars(dev);
1681 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1682 uint32_t offset;
1683
1684 KASSERT(msix->msix_table_len > index, ("bogus index"));
1685 offset = msix->msix_table_offset + index * 16;
1686 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1687 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1688 bus_write_4(msix->msix_table_res, offset + 8, data);
1689 }
1690
1691 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1692 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1693 uint64_t address, uint32_t data)
1694 {
1695
1696 if (pci_msix_rewrite_table) {
1697 struct pci_devinfo *dinfo = device_get_ivars(child);
1698 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1699
1700 /*
1701 * Some VM hosts require MSIX to be disabled in the
1702 * control register before updating the MSIX table
1703 * entries are allowed. It is not enough to only
1704 * disable MSIX while updating a single entry. MSIX
1705 * must be disabled while updating all entries in the
1706 * table.
1707 */
1708 pci_write_config(child,
1709 msix->msix_location + PCIR_MSIX_CTRL,
1710 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1711 pci_resume_msix(child);
1712 } else
1713 pci_write_msix_entry(child, index, address, data);
1714
1715 /* Enable MSI -> HT mapping. */
1716 pci_ht_map_msi(child, address);
1717 }
1718
1719 void
pci_mask_msix(device_t dev,u_int index)1720 pci_mask_msix(device_t dev, u_int index)
1721 {
1722 struct pci_devinfo *dinfo = device_get_ivars(dev);
1723 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1724 uint32_t offset, val;
1725
1726 KASSERT(msix->msix_msgnum > index, ("bogus index"));
1727 offset = msix->msix_table_offset + index * 16 + 12;
1728 val = bus_read_4(msix->msix_table_res, offset);
1729 val |= PCIM_MSIX_VCTRL_MASK;
1730
1731 /*
1732 * Some devices (e.g. Samsung PM961) do not support reads of this
1733 * register, so always write the new value.
1734 */
1735 bus_write_4(msix->msix_table_res, offset, val);
1736 }
1737
1738 void
pci_unmask_msix(device_t dev,u_int index)1739 pci_unmask_msix(device_t dev, u_int index)
1740 {
1741 struct pci_devinfo *dinfo = device_get_ivars(dev);
1742 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1743 uint32_t offset, val;
1744
1745 KASSERT(msix->msix_table_len > index, ("bogus index"));
1746 offset = msix->msix_table_offset + index * 16 + 12;
1747 val = bus_read_4(msix->msix_table_res, offset);
1748 val &= ~PCIM_MSIX_VCTRL_MASK;
1749
1750 /*
1751 * Some devices (e.g. Samsung PM961) do not support reads of this
1752 * register, so always write the new value.
1753 */
1754 bus_write_4(msix->msix_table_res, offset, val);
1755 }
1756
1757 int
pci_pending_msix(device_t dev,u_int index)1758 pci_pending_msix(device_t dev, u_int index)
1759 {
1760 struct pci_devinfo *dinfo = device_get_ivars(dev);
1761 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1762 uint32_t offset, bit;
1763
1764 KASSERT(msix->msix_table_len > index, ("bogus index"));
1765 offset = msix->msix_pba_offset + (index / 32) * 4;
1766 bit = 1 << index % 32;
1767 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1768 }
1769
1770 /*
1771 * Restore MSI-X registers and table during resume. If MSI-X is
1772 * enabled then walk the virtual table to restore the actual MSI-X
1773 * table.
1774 */
1775 static void
pci_resume_msix(device_t dev)1776 pci_resume_msix(device_t dev)
1777 {
1778 struct pci_devinfo *dinfo = device_get_ivars(dev);
1779 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1780 struct msix_table_entry *mte;
1781 struct msix_vector *mv;
1782 int i;
1783
1784 if (msix->msix_alloc > 0) {
1785 /* First, mask all vectors. */
1786 for (i = 0; i < msix->msix_msgnum; i++)
1787 pci_mask_msix(dev, i);
1788
1789 /* Second, program any messages with at least one handler. */
1790 for (i = 0; i < msix->msix_table_len; i++) {
1791 mte = &msix->msix_table[i];
1792 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1793 continue;
1794 mv = &msix->msix_vectors[mte->mte_vector - 1];
1795 pci_write_msix_entry(dev, i, mv->mv_address,
1796 mv->mv_data);
1797 pci_unmask_msix(dev, i);
1798 }
1799 }
1800 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1801 msix->msix_ctrl, 2);
1802 }
1803
1804 /*
1805 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1806 * returned in *count. After this function returns, each message will be
1807 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1808 */
1809 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1810 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1811 {
1812 struct pci_devinfo *dinfo = device_get_ivars(child);
1813 pcicfgregs *cfg = &dinfo->cfg;
1814 struct resource_list_entry *rle;
1815 int actual, error, i, irq, max;
1816
1817 /* Don't let count == 0 get us into trouble. */
1818 if (*count == 0)
1819 return (EINVAL);
1820
1821 /* If rid 0 is allocated, then fail. */
1822 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1823 if (rle != NULL && rle->res != NULL)
1824 return (ENXIO);
1825
1826 /* Already have allocated messages? */
1827 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1828 return (ENXIO);
1829
1830 /* If MSI-X is blacklisted for this system, fail. */
1831 if (pci_msix_blacklisted())
1832 return (ENXIO);
1833
1834 /* MSI-X capability present? */
1835 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1836 return (ENODEV);
1837
1838 /* Make sure the appropriate BARs are mapped. */
1839 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1840 cfg->msix.msix_table_bar);
1841 if (rle == NULL || rle->res == NULL ||
1842 !(rman_get_flags(rle->res) & RF_ACTIVE))
1843 return (ENXIO);
1844 cfg->msix.msix_table_res = rle->res;
1845 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1846 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1847 cfg->msix.msix_pba_bar);
1848 if (rle == NULL || rle->res == NULL ||
1849 !(rman_get_flags(rle->res) & RF_ACTIVE))
1850 return (ENXIO);
1851 }
1852 cfg->msix.msix_pba_res = rle->res;
1853
1854 if (bootverbose)
1855 device_printf(child,
1856 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1857 *count, cfg->msix.msix_msgnum);
1858 max = min(*count, cfg->msix.msix_msgnum);
1859 for (i = 0; i < max; i++) {
1860 /* Allocate a message. */
1861 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1862 if (error) {
1863 if (i == 0)
1864 return (error);
1865 break;
1866 }
1867 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1868 irq, 1);
1869 }
1870 actual = i;
1871
1872 if (bootverbose) {
1873 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1874 if (actual == 1)
1875 device_printf(child, "using IRQ %ju for MSI-X\n",
1876 rle->start);
1877 else {
1878 int run;
1879
1880 /*
1881 * Be fancy and try to print contiguous runs of
1882 * IRQ values as ranges. 'irq' is the previous IRQ.
1883 * 'run' is true if we are in a range.
1884 */
1885 device_printf(child, "using IRQs %ju", rle->start);
1886 irq = rle->start;
1887 run = 0;
1888 for (i = 1; i < actual; i++) {
1889 rle = resource_list_find(&dinfo->resources,
1890 SYS_RES_IRQ, i + 1);
1891
1892 /* Still in a run? */
1893 if (rle->start == irq + 1) {
1894 run = 1;
1895 irq++;
1896 continue;
1897 }
1898
1899 /* Finish previous range. */
1900 if (run) {
1901 printf("-%d", irq);
1902 run = 0;
1903 }
1904
1905 /* Start new range. */
1906 printf(",%ju", rle->start);
1907 irq = rle->start;
1908 }
1909
1910 /* Unfinished range? */
1911 if (run)
1912 printf("-%d", irq);
1913 printf(" for MSI-X\n");
1914 }
1915 }
1916
1917 /* Mask all vectors. */
1918 for (i = 0; i < cfg->msix.msix_msgnum; i++)
1919 pci_mask_msix(child, i);
1920
1921 /* Allocate and initialize vector data and virtual table. */
1922 cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
1923 M_DEVBUF, M_WAITOK | M_ZERO);
1924 cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
1925 M_DEVBUF, M_WAITOK | M_ZERO);
1926 for (i = 0; i < actual; i++) {
1927 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1928 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1929 cfg->msix.msix_table[i].mte_vector = i + 1;
1930 }
1931
1932 /* Update control register to enable MSI-X. */
1933 cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1934 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1935 cfg->msix.msix_ctrl, 2);
1936
1937 /* Update counts of alloc'd messages. */
1938 cfg->msix.msix_alloc = actual;
1939 cfg->msix.msix_table_len = actual;
1940 *count = actual;
1941 return (0);
1942 }
1943
1944 /*
1945 * By default, pci_alloc_msix() will assign the allocated IRQ
1946 * resources consecutively to the first N messages in the MSI-X table.
1947 * However, device drivers may want to use different layouts if they
1948 * either receive fewer messages than they asked for, or they wish to
1949 * populate the MSI-X table sparsely. This method allows the driver
1950 * to specify what layout it wants. It must be called after a
1951 * successful pci_alloc_msix() but before any of the associated
1952 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1953 *
1954 * The 'vectors' array contains 'count' message vectors. The array
1955 * maps directly to the MSI-X table in that index 0 in the array
1956 * specifies the vector for the first message in the MSI-X table, etc.
1957 * The vector value in each array index can either be 0 to indicate
1958 * that no vector should be assigned to a message slot, or it can be a
1959 * number from 1 to N (where N is the count returned from a
1960 * succcessful call to pci_alloc_msix()) to indicate which message
1961 * vector (IRQ) to be used for the corresponding message.
1962 *
1963 * On successful return, each message with a non-zero vector will have
1964 * an associated SYS_RES_IRQ whose rid is equal to the array index +
1965 * 1. Additionally, if any of the IRQs allocated via the previous
1966 * call to pci_alloc_msix() are not used in the mapping, those IRQs
1967 * will be freed back to the system automatically.
1968 *
1969 * For example, suppose a driver has a MSI-X table with 6 messages and
1970 * asks for 6 messages, but pci_alloc_msix() only returns a count of
1971 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
1972 * C. After the call to pci_alloc_msix(), the device will be setup to
1973 * have an MSI-X table of ABC--- (where - means no vector assigned).
1974 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
1975 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
1976 * be freed back to the system. This device will also have valid
1977 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
1978 *
1979 * In any case, the SYS_RES_IRQ rid X will always map to the message
1980 * at MSI-X table index X - 1 and will only be valid if a vector is
1981 * assigned to that table entry.
1982 */
1983 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)1984 pci_remap_msix_method(device_t dev, device_t child, int count,
1985 const u_int *vectors)
1986 {
1987 struct pci_devinfo *dinfo = device_get_ivars(child);
1988 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1989 struct resource_list_entry *rle;
1990 int i, irq, j, *used;
1991
1992 /*
1993 * Have to have at least one message in the table but the
1994 * table can't be bigger than the actual MSI-X table in the
1995 * device.
1996 */
1997 if (count == 0 || count > msix->msix_msgnum)
1998 return (EINVAL);
1999
2000 /* Sanity check the vectors. */
2001 for (i = 0; i < count; i++)
2002 if (vectors[i] > msix->msix_alloc)
2003 return (EINVAL);
2004
2005 /*
2006 * Make sure there aren't any holes in the vectors to be used.
2007 * It's a big pain to support it, and it doesn't really make
2008 * sense anyway. Also, at least one vector must be used.
2009 */
2010 used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK |
2011 M_ZERO);
2012 for (i = 0; i < count; i++)
2013 if (vectors[i] != 0)
2014 used[vectors[i] - 1] = 1;
2015 for (i = 0; i < msix->msix_alloc - 1; i++)
2016 if (used[i] == 0 && used[i + 1] == 1) {
2017 free(used, M_DEVBUF);
2018 return (EINVAL);
2019 }
2020 if (used[0] != 1) {
2021 free(used, M_DEVBUF);
2022 return (EINVAL);
2023 }
2024
2025 /* Make sure none of the resources are allocated. */
2026 for (i = 0; i < msix->msix_table_len; i++) {
2027 if (msix->msix_table[i].mte_vector == 0)
2028 continue;
2029 if (msix->msix_table[i].mte_handlers > 0) {
2030 free(used, M_DEVBUF);
2031 return (EBUSY);
2032 }
2033 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2034 KASSERT(rle != NULL, ("missing resource"));
2035 if (rle->res != NULL) {
2036 free(used, M_DEVBUF);
2037 return (EBUSY);
2038 }
2039 }
2040
2041 /* Free the existing resource list entries. */
2042 for (i = 0; i < msix->msix_table_len; i++) {
2043 if (msix->msix_table[i].mte_vector == 0)
2044 continue;
2045 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2046 }
2047
2048 /*
2049 * Build the new virtual table keeping track of which vectors are
2050 * used.
2051 */
2052 free(msix->msix_table, M_DEVBUF);
2053 msix->msix_table = malloc(sizeof(struct msix_table_entry) * count,
2054 M_DEVBUF, M_WAITOK | M_ZERO);
2055 for (i = 0; i < count; i++)
2056 msix->msix_table[i].mte_vector = vectors[i];
2057 msix->msix_table_len = count;
2058
2059 /* Free any unused IRQs and resize the vectors array if necessary. */
2060 j = msix->msix_alloc - 1;
2061 if (used[j] == 0) {
2062 struct msix_vector *vec;
2063
2064 while (used[j] == 0) {
2065 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2066 msix->msix_vectors[j].mv_irq);
2067 j--;
2068 }
2069 vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF,
2070 M_WAITOK);
2071 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2072 (j + 1));
2073 free(msix->msix_vectors, M_DEVBUF);
2074 msix->msix_vectors = vec;
2075 msix->msix_alloc = j + 1;
2076 }
2077 free(used, M_DEVBUF);
2078
2079 /* Map the IRQs onto the rids. */
2080 for (i = 0; i < count; i++) {
2081 if (vectors[i] == 0)
2082 continue;
2083 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2084 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2085 irq, 1);
2086 }
2087
2088 if (bootverbose) {
2089 device_printf(child, "Remapped MSI-X IRQs as: ");
2090 for (i = 0; i < count; i++) {
2091 if (i != 0)
2092 printf(", ");
2093 if (vectors[i] == 0)
2094 printf("---");
2095 else
2096 printf("%d",
2097 msix->msix_vectors[vectors[i] - 1].mv_irq);
2098 }
2099 printf("\n");
2100 }
2101
2102 return (0);
2103 }
2104
2105 static int
pci_release_msix(device_t dev,device_t child)2106 pci_release_msix(device_t dev, device_t child)
2107 {
2108 struct pci_devinfo *dinfo = device_get_ivars(child);
2109 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2110 struct resource_list_entry *rle;
2111 int i;
2112
2113 /* Do we have any messages to release? */
2114 if (msix->msix_alloc == 0)
2115 return (ENODEV);
2116
2117 /* Make sure none of the resources are allocated. */
2118 for (i = 0; i < msix->msix_table_len; i++) {
2119 if (msix->msix_table[i].mte_vector == 0)
2120 continue;
2121 if (msix->msix_table[i].mte_handlers > 0)
2122 return (EBUSY);
2123 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2124 KASSERT(rle != NULL, ("missing resource"));
2125 if (rle->res != NULL)
2126 return (EBUSY);
2127 }
2128
2129 /* Update control register to disable MSI-X. */
2130 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2131 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2132 msix->msix_ctrl, 2);
2133
2134 /* Free the resource list entries. */
2135 for (i = 0; i < msix->msix_table_len; i++) {
2136 if (msix->msix_table[i].mte_vector == 0)
2137 continue;
2138 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2139 }
2140 free(msix->msix_table, M_DEVBUF);
2141 msix->msix_table_len = 0;
2142
2143 /* Release the IRQs. */
2144 for (i = 0; i < msix->msix_alloc; i++)
2145 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2146 msix->msix_vectors[i].mv_irq);
2147 free(msix->msix_vectors, M_DEVBUF);
2148 msix->msix_alloc = 0;
2149 return (0);
2150 }
2151
2152 /*
2153 * Return the max supported MSI-X messages this device supports.
2154 * Basically, assuming the MD code can alloc messages, this function
2155 * should return the maximum value that pci_alloc_msix() can return.
2156 * Thus, it is subject to the tunables, etc.
2157 */
2158 int
pci_msix_count_method(device_t dev,device_t child)2159 pci_msix_count_method(device_t dev, device_t child)
2160 {
2161 struct pci_devinfo *dinfo = device_get_ivars(child);
2162 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2163
2164 if (pci_do_msix && msix->msix_location != 0)
2165 return (msix->msix_msgnum);
2166 return (0);
2167 }
2168
2169 int
pci_msix_pba_bar_method(device_t dev,device_t child)2170 pci_msix_pba_bar_method(device_t dev, device_t child)
2171 {
2172 struct pci_devinfo *dinfo = device_get_ivars(child);
2173 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2174
2175 if (pci_do_msix && msix->msix_location != 0)
2176 return (msix->msix_pba_bar);
2177 return (-1);
2178 }
2179
2180 int
pci_msix_table_bar_method(device_t dev,device_t child)2181 pci_msix_table_bar_method(device_t dev, device_t child)
2182 {
2183 struct pci_devinfo *dinfo = device_get_ivars(child);
2184 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2185
2186 if (pci_do_msix && msix->msix_location != 0)
2187 return (msix->msix_table_bar);
2188 return (-1);
2189 }
2190
2191 /*
2192 * HyperTransport MSI mapping control
2193 */
2194 void
pci_ht_map_msi(device_t dev,uint64_t addr)2195 pci_ht_map_msi(device_t dev, uint64_t addr)
2196 {
2197 struct pci_devinfo *dinfo = device_get_ivars(dev);
2198 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2199
2200 if (!ht->ht_msimap)
2201 return;
2202
2203 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2204 ht->ht_msiaddr >> 20 == addr >> 20) {
2205 /* Enable MSI -> HT mapping. */
2206 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2207 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2208 ht->ht_msictrl, 2);
2209 }
2210
2211 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2212 /* Disable MSI -> HT mapping. */
2213 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2214 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2215 ht->ht_msictrl, 2);
2216 }
2217 }
2218
2219 int
pci_get_relaxed_ordering_enabled(device_t dev)2220 pci_get_relaxed_ordering_enabled(device_t dev)
2221 {
2222 struct pci_devinfo *dinfo = device_get_ivars(dev);
2223 int cap;
2224 uint16_t val;
2225
2226 cap = dinfo->cfg.pcie.pcie_location;
2227 if (cap == 0)
2228 return (0);
2229 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2230 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2231 return (val != 0);
2232 }
2233
2234 int
pci_get_max_payload(device_t dev)2235 pci_get_max_payload(device_t dev)
2236 {
2237 struct pci_devinfo *dinfo = device_get_ivars(dev);
2238 int cap;
2239 uint16_t val;
2240
2241 cap = dinfo->cfg.pcie.pcie_location;
2242 if (cap == 0)
2243 return (0);
2244 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2245 val &= PCIEM_CTL_MAX_PAYLOAD;
2246 val >>= 5;
2247 return (1 << (val + 7));
2248 }
2249
2250 int
pci_get_max_read_req(device_t dev)2251 pci_get_max_read_req(device_t dev)
2252 {
2253 struct pci_devinfo *dinfo = device_get_ivars(dev);
2254 int cap;
2255 uint16_t val;
2256
2257 cap = dinfo->cfg.pcie.pcie_location;
2258 if (cap == 0)
2259 return (0);
2260 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2261 val &= PCIEM_CTL_MAX_READ_REQUEST;
2262 val >>= 12;
2263 return (1 << (val + 7));
2264 }
2265
2266 int
pci_set_max_read_req(device_t dev,int size)2267 pci_set_max_read_req(device_t dev, int size)
2268 {
2269 struct pci_devinfo *dinfo = device_get_ivars(dev);
2270 int cap;
2271 uint16_t val;
2272
2273 cap = dinfo->cfg.pcie.pcie_location;
2274 if (cap == 0)
2275 return (0);
2276 if (size < 128)
2277 size = 128;
2278 if (size > 4096)
2279 size = 4096;
2280 size = (1 << (fls(size) - 1));
2281 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2282 val &= ~PCIEM_CTL_MAX_READ_REQUEST;
2283 val |= (fls(size) - 8) << 12;
2284 pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2);
2285 return (size);
2286 }
2287
2288 uint32_t
pcie_read_config(device_t dev,int reg,int width)2289 pcie_read_config(device_t dev, int reg, int width)
2290 {
2291 struct pci_devinfo *dinfo = device_get_ivars(dev);
2292 int cap;
2293
2294 cap = dinfo->cfg.pcie.pcie_location;
2295 if (cap == 0) {
2296 if (width == 2)
2297 return (0xffff);
2298 return (0xffffffff);
2299 }
2300
2301 return (pci_read_config(dev, cap + reg, width));
2302 }
2303
2304 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2305 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2306 {
2307 struct pci_devinfo *dinfo = device_get_ivars(dev);
2308 int cap;
2309
2310 cap = dinfo->cfg.pcie.pcie_location;
2311 if (cap == 0)
2312 return;
2313 pci_write_config(dev, cap + reg, value, width);
2314 }
2315
2316 /*
2317 * Adjusts a PCI-e capability register by clearing the bits in mask
2318 * and setting the bits in (value & mask). Bits not set in mask are
2319 * not adjusted.
2320 *
2321 * Returns the old value on success or all ones on failure.
2322 */
2323 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2324 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2325 int width)
2326 {
2327 struct pci_devinfo *dinfo = device_get_ivars(dev);
2328 uint32_t old, new;
2329 int cap;
2330
2331 cap = dinfo->cfg.pcie.pcie_location;
2332 if (cap == 0) {
2333 if (width == 2)
2334 return (0xffff);
2335 return (0xffffffff);
2336 }
2337
2338 old = pci_read_config(dev, cap + reg, width);
2339 new = old & ~mask;
2340 new |= (value & mask);
2341 pci_write_config(dev, cap + reg, new, width);
2342 return (old);
2343 }
2344
2345 /*
2346 * Support for MSI message signalled interrupts.
2347 */
2348 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2349 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2350 uint16_t data)
2351 {
2352 struct pci_devinfo *dinfo = device_get_ivars(child);
2353 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2354
2355 /* Write data and address values. */
2356 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2357 address & 0xffffffff, 4);
2358 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2359 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2360 address >> 32, 4);
2361 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2362 data, 2);
2363 } else
2364 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2365 2);
2366
2367 /* Enable MSI in the control register. */
2368 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2369 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2370 msi->msi_ctrl, 2);
2371
2372 /* Enable MSI -> HT mapping. */
2373 pci_ht_map_msi(child, address);
2374 }
2375
2376 void
pci_disable_msi_method(device_t dev,device_t child)2377 pci_disable_msi_method(device_t dev, device_t child)
2378 {
2379 struct pci_devinfo *dinfo = device_get_ivars(child);
2380 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2381
2382 /* Disable MSI -> HT mapping. */
2383 pci_ht_map_msi(child, 0);
2384
2385 /* Disable MSI in the control register. */
2386 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2387 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2388 msi->msi_ctrl, 2);
2389 }
2390
2391 /*
2392 * Restore MSI registers during resume. If MSI is enabled then
2393 * restore the data and address registers in addition to the control
2394 * register.
2395 */
2396 static void
pci_resume_msi(device_t dev)2397 pci_resume_msi(device_t dev)
2398 {
2399 struct pci_devinfo *dinfo = device_get_ivars(dev);
2400 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2401 uint64_t address;
2402 uint16_t data;
2403
2404 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2405 address = msi->msi_addr;
2406 data = msi->msi_data;
2407 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2408 address & 0xffffffff, 4);
2409 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2410 pci_write_config(dev, msi->msi_location +
2411 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2412 pci_write_config(dev, msi->msi_location +
2413 PCIR_MSI_DATA_64BIT, data, 2);
2414 } else
2415 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2416 data, 2);
2417 }
2418 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2419 2);
2420 }
2421
2422 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2423 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2424 {
2425 struct pci_devinfo *dinfo = device_get_ivars(dev);
2426 pcicfgregs *cfg = &dinfo->cfg;
2427 struct resource_list_entry *rle;
2428 struct msix_table_entry *mte;
2429 struct msix_vector *mv;
2430 uint64_t addr;
2431 uint32_t data;
2432 int error, i, j;
2433
2434 /*
2435 * Handle MSI first. We try to find this IRQ among our list
2436 * of MSI IRQs. If we find it, we request updated address and
2437 * data registers and apply the results.
2438 */
2439 if (cfg->msi.msi_alloc > 0) {
2440 /* If we don't have any active handlers, nothing to do. */
2441 if (cfg->msi.msi_handlers == 0)
2442 return (0);
2443 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2444 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2445 i + 1);
2446 if (rle->start == irq) {
2447 error = PCIB_MAP_MSI(device_get_parent(bus),
2448 dev, irq, &addr, &data);
2449 if (error)
2450 return (error);
2451 pci_disable_msi(dev);
2452 dinfo->cfg.msi.msi_addr = addr;
2453 dinfo->cfg.msi.msi_data = data;
2454 pci_enable_msi(dev, addr, data);
2455 return (0);
2456 }
2457 }
2458 return (ENOENT);
2459 }
2460
2461 /*
2462 * For MSI-X, we check to see if we have this IRQ. If we do,
2463 * we request the updated mapping info. If that works, we go
2464 * through all the slots that use this IRQ and update them.
2465 */
2466 if (cfg->msix.msix_alloc > 0) {
2467 bool found = false;
2468
2469 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2470 mv = &cfg->msix.msix_vectors[i];
2471 if (mv->mv_irq == irq) {
2472 error = PCIB_MAP_MSI(device_get_parent(bus),
2473 dev, irq, &addr, &data);
2474 if (error)
2475 return (error);
2476 mv->mv_address = addr;
2477 mv->mv_data = data;
2478 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2479 mte = &cfg->msix.msix_table[j];
2480 if (mte->mte_vector != i + 1)
2481 continue;
2482 if (mte->mte_handlers == 0)
2483 continue;
2484 pci_mask_msix(dev, j);
2485 pci_enable_msix(dev, j, addr, data);
2486 pci_unmask_msix(dev, j);
2487 }
2488 found = true;
2489 }
2490 }
2491 return (found ? 0 : ENOENT);
2492 }
2493
2494 return (ENOENT);
2495 }
2496
2497 /*
2498 * Returns true if the specified device is blacklisted because MSI
2499 * doesn't work.
2500 */
2501 int
pci_msi_device_blacklisted(device_t dev)2502 pci_msi_device_blacklisted(device_t dev)
2503 {
2504
2505 if (!pci_honor_msi_blacklist)
2506 return (0);
2507
2508 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2509 }
2510
2511 /*
2512 * Determine if MSI is blacklisted globally on this system. Currently,
2513 * we just check for blacklisted chipsets as represented by the
2514 * host-PCI bridge at device 0:0:0. In the future, it may become
2515 * necessary to check other system attributes, such as the kenv values
2516 * that give the motherboard manufacturer and model number.
2517 */
2518 static int
pci_msi_blacklisted(void)2519 pci_msi_blacklisted(void)
2520 {
2521 device_t dev;
2522
2523 if (!pci_honor_msi_blacklist)
2524 return (0);
2525
2526 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2527 if (!(pcie_chipset || pcix_chipset)) {
2528 if (vm_guest != VM_GUEST_NO) {
2529 /*
2530 * Whitelist older chipsets in virtual
2531 * machines known to support MSI.
2532 */
2533 dev = pci_find_bsf(0, 0, 0);
2534 if (dev != NULL)
2535 return (!pci_has_quirk(pci_get_devid(dev),
2536 PCI_QUIRK_ENABLE_MSI_VM));
2537 }
2538 return (1);
2539 }
2540
2541 dev = pci_find_bsf(0, 0, 0);
2542 if (dev != NULL)
2543 return (pci_msi_device_blacklisted(dev));
2544 return (0);
2545 }
2546
2547 /*
2548 * Returns true if the specified device is blacklisted because MSI-X
2549 * doesn't work. Note that this assumes that if MSI doesn't work,
2550 * MSI-X doesn't either.
2551 */
2552 int
pci_msix_device_blacklisted(device_t dev)2553 pci_msix_device_blacklisted(device_t dev)
2554 {
2555
2556 if (!pci_honor_msi_blacklist)
2557 return (0);
2558
2559 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2560 return (1);
2561
2562 return (pci_msi_device_blacklisted(dev));
2563 }
2564
2565 /*
2566 * Determine if MSI-X is blacklisted globally on this system. If MSI
2567 * is blacklisted, assume that MSI-X is as well. Check for additional
2568 * chipsets where MSI works but MSI-X does not.
2569 */
2570 static int
pci_msix_blacklisted(void)2571 pci_msix_blacklisted(void)
2572 {
2573 device_t dev;
2574
2575 if (!pci_honor_msi_blacklist)
2576 return (0);
2577
2578 dev = pci_find_bsf(0, 0, 0);
2579 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2580 PCI_QUIRK_DISABLE_MSIX))
2581 return (1);
2582
2583 return (pci_msi_blacklisted());
2584 }
2585
2586 /*
2587 * Attempt to allocate *count MSI messages. The actual number allocated is
2588 * returned in *count. After this function returns, each message will be
2589 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2590 */
2591 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2592 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2593 {
2594 struct pci_devinfo *dinfo = device_get_ivars(child);
2595 pcicfgregs *cfg = &dinfo->cfg;
2596 struct resource_list_entry *rle;
2597 int actual, error, i, irqs[32];
2598 uint16_t ctrl;
2599
2600 /* Don't let count == 0 get us into trouble. */
2601 if (*count == 0)
2602 return (EINVAL);
2603
2604 /* If rid 0 is allocated, then fail. */
2605 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2606 if (rle != NULL && rle->res != NULL)
2607 return (ENXIO);
2608
2609 /* Already have allocated messages? */
2610 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2611 return (ENXIO);
2612
2613 /* If MSI is blacklisted for this system, fail. */
2614 if (pci_msi_blacklisted())
2615 return (ENXIO);
2616
2617 /* MSI capability present? */
2618 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2619 return (ENODEV);
2620
2621 if (bootverbose)
2622 device_printf(child,
2623 "attempting to allocate %d MSI vectors (%d supported)\n",
2624 *count, cfg->msi.msi_msgnum);
2625
2626 /* Don't ask for more than the device supports. */
2627 actual = min(*count, cfg->msi.msi_msgnum);
2628
2629 /* Don't ask for more than 32 messages. */
2630 actual = min(actual, 32);
2631
2632 /* MSI requires power of 2 number of messages. */
2633 if (!powerof2(actual))
2634 return (EINVAL);
2635
2636 for (;;) {
2637 /* Try to allocate N messages. */
2638 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2639 actual, irqs);
2640 if (error == 0)
2641 break;
2642 if (actual == 1)
2643 return (error);
2644
2645 /* Try N / 2. */
2646 actual >>= 1;
2647 }
2648
2649 /*
2650 * We now have N actual messages mapped onto SYS_RES_IRQ
2651 * resources in the irqs[] array, so add new resources
2652 * starting at rid 1.
2653 */
2654 for (i = 0; i < actual; i++)
2655 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2656 irqs[i], irqs[i], 1);
2657
2658 if (bootverbose) {
2659 if (actual == 1)
2660 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2661 else {
2662 int run;
2663
2664 /*
2665 * Be fancy and try to print contiguous runs
2666 * of IRQ values as ranges. 'run' is true if
2667 * we are in a range.
2668 */
2669 device_printf(child, "using IRQs %d", irqs[0]);
2670 run = 0;
2671 for (i = 1; i < actual; i++) {
2672 /* Still in a run? */
2673 if (irqs[i] == irqs[i - 1] + 1) {
2674 run = 1;
2675 continue;
2676 }
2677
2678 /* Finish previous range. */
2679 if (run) {
2680 printf("-%d", irqs[i - 1]);
2681 run = 0;
2682 }
2683
2684 /* Start new range. */
2685 printf(",%d", irqs[i]);
2686 }
2687
2688 /* Unfinished range? */
2689 if (run)
2690 printf("-%d", irqs[actual - 1]);
2691 printf(" for MSI\n");
2692 }
2693 }
2694
2695 /* Update control register with actual count. */
2696 ctrl = cfg->msi.msi_ctrl;
2697 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2698 ctrl |= (ffs(actual) - 1) << 4;
2699 cfg->msi.msi_ctrl = ctrl;
2700 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2701
2702 /* Update counts of alloc'd messages. */
2703 cfg->msi.msi_alloc = actual;
2704 cfg->msi.msi_handlers = 0;
2705 *count = actual;
2706 return (0);
2707 }
2708
2709 /* Release the MSI messages associated with this device. */
2710 int
pci_release_msi_method(device_t dev,device_t child)2711 pci_release_msi_method(device_t dev, device_t child)
2712 {
2713 struct pci_devinfo *dinfo = device_get_ivars(child);
2714 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2715 struct resource_list_entry *rle;
2716 int error, i, irqs[32];
2717
2718 /* Try MSI-X first. */
2719 error = pci_release_msix(dev, child);
2720 if (error != ENODEV)
2721 return (error);
2722
2723 /* Do we have any messages to release? */
2724 if (msi->msi_alloc == 0)
2725 return (ENODEV);
2726 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2727
2728 /* Make sure none of the resources are allocated. */
2729 if (msi->msi_handlers > 0)
2730 return (EBUSY);
2731 for (i = 0; i < msi->msi_alloc; i++) {
2732 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2733 KASSERT(rle != NULL, ("missing MSI resource"));
2734 if (rle->res != NULL)
2735 return (EBUSY);
2736 irqs[i] = rle->start;
2737 }
2738
2739 /* Update control register with 0 count. */
2740 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2741 ("%s: MSI still enabled", __func__));
2742 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2743 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2744 msi->msi_ctrl, 2);
2745
2746 /* Release the messages. */
2747 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2748 for (i = 0; i < msi->msi_alloc; i++)
2749 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2750
2751 /* Update alloc count. */
2752 msi->msi_alloc = 0;
2753 msi->msi_addr = 0;
2754 msi->msi_data = 0;
2755 return (0);
2756 }
2757
2758 /*
2759 * Return the max supported MSI messages this device supports.
2760 * Basically, assuming the MD code can alloc messages, this function
2761 * should return the maximum value that pci_alloc_msi() can return.
2762 * Thus, it is subject to the tunables, etc.
2763 */
2764 int
pci_msi_count_method(device_t dev,device_t child)2765 pci_msi_count_method(device_t dev, device_t child)
2766 {
2767 struct pci_devinfo *dinfo = device_get_ivars(child);
2768 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2769
2770 if (pci_do_msi && msi->msi_location != 0)
2771 return (msi->msi_msgnum);
2772 return (0);
2773 }
2774
2775 /* free pcicfgregs structure and all depending data structures */
2776
2777 int
pci_freecfg(struct pci_devinfo * dinfo)2778 pci_freecfg(struct pci_devinfo *dinfo)
2779 {
2780 struct devlist *devlist_head;
2781 struct pci_map *pm, *next;
2782 int i;
2783
2784 devlist_head = &pci_devq;
2785
2786 if (dinfo->cfg.vpd.vpd_reg) {
2787 free(dinfo->cfg.vpd.vpd_ident, M_DEVBUF);
2788 for (i = 0; i < dinfo->cfg.vpd.vpd_rocnt; i++)
2789 free(dinfo->cfg.vpd.vpd_ros[i].value, M_DEVBUF);
2790 free(dinfo->cfg.vpd.vpd_ros, M_DEVBUF);
2791 for (i = 0; i < dinfo->cfg.vpd.vpd_wcnt; i++)
2792 free(dinfo->cfg.vpd.vpd_w[i].value, M_DEVBUF);
2793 free(dinfo->cfg.vpd.vpd_w, M_DEVBUF);
2794 }
2795 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2796 free(pm, M_DEVBUF);
2797 }
2798 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2799 free(dinfo, M_DEVBUF);
2800
2801 /* increment the generation count */
2802 pci_generation++;
2803
2804 /* we're losing one device */
2805 pci_numdevs--;
2806 return (0);
2807 }
2808
2809 /*
2810 * PCI power manangement
2811 */
2812 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2813 pci_set_powerstate_method(device_t dev, device_t child, int state)
2814 {
2815 struct pci_devinfo *dinfo = device_get_ivars(child);
2816 pcicfgregs *cfg = &dinfo->cfg;
2817 uint16_t status;
2818 int oldstate, highest, delay;
2819
2820 if (cfg->pp.pp_cap == 0)
2821 return (EOPNOTSUPP);
2822
2823 /*
2824 * Optimize a no state change request away. While it would be OK to
2825 * write to the hardware in theory, some devices have shown odd
2826 * behavior when going from D3 -> D3.
2827 */
2828 oldstate = pci_get_powerstate(child);
2829 if (oldstate == state)
2830 return (0);
2831
2832 /*
2833 * The PCI power management specification states that after a state
2834 * transition between PCI power states, system software must
2835 * guarantee a minimal delay before the function accesses the device.
2836 * Compute the worst case delay that we need to guarantee before we
2837 * access the device. Many devices will be responsive much more
2838 * quickly than this delay, but there are some that don't respond
2839 * instantly to state changes. Transitions to/from D3 state require
2840 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2841 * is done below with DELAY rather than a sleeper function because
2842 * this function can be called from contexts where we cannot sleep.
2843 */
2844 highest = (oldstate > state) ? oldstate : state;
2845 if (highest == PCI_POWERSTATE_D3)
2846 delay = 10000;
2847 else if (highest == PCI_POWERSTATE_D2)
2848 delay = 200;
2849 else
2850 delay = 0;
2851 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2)
2852 & ~PCIM_PSTAT_DMASK;
2853 switch (state) {
2854 case PCI_POWERSTATE_D0:
2855 status |= PCIM_PSTAT_D0;
2856 break;
2857 case PCI_POWERSTATE_D1:
2858 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2859 return (EOPNOTSUPP);
2860 status |= PCIM_PSTAT_D1;
2861 break;
2862 case PCI_POWERSTATE_D2:
2863 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2864 return (EOPNOTSUPP);
2865 status |= PCIM_PSTAT_D2;
2866 break;
2867 case PCI_POWERSTATE_D3:
2868 status |= PCIM_PSTAT_D3;
2869 break;
2870 default:
2871 return (EINVAL);
2872 }
2873
2874 if (bootverbose)
2875 pci_printf(cfg, "Transition from D%d to D%d\n", oldstate,
2876 state);
2877
2878 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2);
2879 if (delay)
2880 DELAY(delay);
2881 return (0);
2882 }
2883
2884 int
pci_get_powerstate_method(device_t dev,device_t child)2885 pci_get_powerstate_method(device_t dev, device_t child)
2886 {
2887 struct pci_devinfo *dinfo = device_get_ivars(child);
2888 pcicfgregs *cfg = &dinfo->cfg;
2889 uint16_t status;
2890 int result;
2891
2892 if (cfg->pp.pp_cap != 0) {
2893 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2);
2894 switch (status & PCIM_PSTAT_DMASK) {
2895 case PCIM_PSTAT_D0:
2896 result = PCI_POWERSTATE_D0;
2897 break;
2898 case PCIM_PSTAT_D1:
2899 result = PCI_POWERSTATE_D1;
2900 break;
2901 case PCIM_PSTAT_D2:
2902 result = PCI_POWERSTATE_D2;
2903 break;
2904 case PCIM_PSTAT_D3:
2905 result = PCI_POWERSTATE_D3;
2906 break;
2907 default:
2908 result = PCI_POWERSTATE_UNKNOWN;
2909 break;
2910 }
2911 } else {
2912 /* No support, device is always at D0 */
2913 result = PCI_POWERSTATE_D0;
2914 }
2915 return (result);
2916 }
2917
2918 /*
2919 * Some convenience functions for PCI device drivers.
2920 */
2921
2922 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)2923 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
2924 {
2925 uint16_t command;
2926
2927 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2928 command |= bit;
2929 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2930 }
2931
2932 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)2933 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
2934 {
2935 uint16_t command;
2936
2937 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2938 command &= ~bit;
2939 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2940 }
2941
2942 int
pci_enable_busmaster_method(device_t dev,device_t child)2943 pci_enable_busmaster_method(device_t dev, device_t child)
2944 {
2945 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2946 return (0);
2947 }
2948
2949 int
pci_disable_busmaster_method(device_t dev,device_t child)2950 pci_disable_busmaster_method(device_t dev, device_t child)
2951 {
2952 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2953 return (0);
2954 }
2955
2956 int
pci_enable_io_method(device_t dev,device_t child,int space)2957 pci_enable_io_method(device_t dev, device_t child, int space)
2958 {
2959 uint16_t bit;
2960
2961 switch(space) {
2962 case SYS_RES_IOPORT:
2963 bit = PCIM_CMD_PORTEN;
2964 break;
2965 case SYS_RES_MEMORY:
2966 bit = PCIM_CMD_MEMEN;
2967 break;
2968 default:
2969 return (EINVAL);
2970 }
2971 pci_set_command_bit(dev, child, bit);
2972 return (0);
2973 }
2974
2975 int
pci_disable_io_method(device_t dev,device_t child,int space)2976 pci_disable_io_method(device_t dev, device_t child, int space)
2977 {
2978 uint16_t bit;
2979
2980 switch(space) {
2981 case SYS_RES_IOPORT:
2982 bit = PCIM_CMD_PORTEN;
2983 break;
2984 case SYS_RES_MEMORY:
2985 bit = PCIM_CMD_MEMEN;
2986 break;
2987 default:
2988 return (EINVAL);
2989 }
2990 pci_clear_command_bit(dev, child, bit);
2991 return (0);
2992 }
2993
2994 /*
2995 * New style pci driver. Parent device is either a pci-host-bridge or a
2996 * pci-pci-bridge. Both kinds are represented by instances of pcib.
2997 */
2998
2999 void
pci_print_verbose(struct pci_devinfo * dinfo)3000 pci_print_verbose(struct pci_devinfo *dinfo)
3001 {
3002
3003 if (bootverbose) {
3004 pcicfgregs *cfg = &dinfo->cfg;
3005
3006 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
3007 cfg->vendor, cfg->device, cfg->revid);
3008 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
3009 cfg->domain, cfg->bus, cfg->slot, cfg->func);
3010 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
3011 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
3012 cfg->mfdev);
3013 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
3014 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
3015 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
3016 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
3017 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
3018 if (cfg->intpin > 0)
3019 printf("\tintpin=%c, irq=%d\n",
3020 cfg->intpin +'a' -1, cfg->intline);
3021 if (cfg->pp.pp_cap) {
3022 uint16_t status;
3023
3024 status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2);
3025 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3026 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3027 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3028 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3029 status & PCIM_PSTAT_DMASK);
3030 }
3031 if (cfg->msi.msi_location) {
3032 int ctrl;
3033
3034 ctrl = cfg->msi.msi_ctrl;
3035 printf("\tMSI supports %d message%s%s%s\n",
3036 cfg->msi.msi_msgnum,
3037 (cfg->msi.msi_msgnum == 1) ? "" : "s",
3038 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3039 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3040 }
3041 if (cfg->msix.msix_location) {
3042 printf("\tMSI-X supports %d message%s ",
3043 cfg->msix.msix_msgnum,
3044 (cfg->msix.msix_msgnum == 1) ? "" : "s");
3045 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3046 printf("in map 0x%x\n",
3047 cfg->msix.msix_table_bar);
3048 else
3049 printf("in maps 0x%x and 0x%x\n",
3050 cfg->msix.msix_table_bar,
3051 cfg->msix.msix_pba_bar);
3052 }
3053 }
3054 }
3055
3056 static int
pci_porten(device_t dev)3057 pci_porten(device_t dev)
3058 {
3059 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3060 }
3061
3062 static int
pci_memen(device_t dev)3063 pci_memen(device_t dev)
3064 {
3065 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3066 }
3067
3068 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3069 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3070 int *bar64)
3071 {
3072 struct pci_devinfo *dinfo;
3073 pci_addr_t map, testval;
3074 int ln2range;
3075 uint16_t cmd;
3076
3077 /*
3078 * The device ROM BAR is special. It is always a 32-bit
3079 * memory BAR. Bit 0 is special and should not be set when
3080 * sizing the BAR.
3081 */
3082 dinfo = device_get_ivars(dev);
3083 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3084 map = pci_read_config(dev, reg, 4);
3085 pci_write_config(dev, reg, 0xfffffffe, 4);
3086 testval = pci_read_config(dev, reg, 4);
3087 pci_write_config(dev, reg, map, 4);
3088 *mapp = map;
3089 *testvalp = testval;
3090 if (bar64 != NULL)
3091 *bar64 = 0;
3092 return;
3093 }
3094
3095 map = pci_read_config(dev, reg, 4);
3096 ln2range = pci_maprange(map);
3097 if (ln2range == 64)
3098 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3099
3100 /*
3101 * Disable decoding via the command register before
3102 * determining the BAR's length since we will be placing it in
3103 * a weird state.
3104 */
3105 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3106 pci_write_config(dev, PCIR_COMMAND,
3107 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3108
3109 /*
3110 * Determine the BAR's length by writing all 1's. The bottom
3111 * log_2(size) bits of the BAR will stick as 0 when we read
3112 * the value back.
3113 *
3114 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3115 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3116 * and combines the result into a 64-bit value." (section 6.2.5.1)
3117 *
3118 * Writes to both registers must be performed before attempting to
3119 * read back the size value.
3120 */
3121 testval = 0;
3122 pci_write_config(dev, reg, 0xffffffff, 4);
3123 if (ln2range == 64) {
3124 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3125 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3126 }
3127 testval |= pci_read_config(dev, reg, 4);
3128
3129 /*
3130 * Restore the original value of the BAR. We may have reprogrammed
3131 * the BAR of the low-level console device and when booting verbose,
3132 * we need the console device addressable.
3133 */
3134 pci_write_config(dev, reg, map, 4);
3135 if (ln2range == 64)
3136 pci_write_config(dev, reg + 4, map >> 32, 4);
3137 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3138
3139 *mapp = map;
3140 *testvalp = testval;
3141 if (bar64 != NULL)
3142 *bar64 = (ln2range == 64);
3143 }
3144
3145 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3146 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3147 {
3148 struct pci_devinfo *dinfo;
3149 int ln2range;
3150
3151 /* The device ROM BAR is always a 32-bit memory BAR. */
3152 dinfo = device_get_ivars(dev);
3153 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3154 ln2range = 32;
3155 else
3156 ln2range = pci_maprange(pm->pm_value);
3157 pci_write_config(dev, pm->pm_reg, base, 4);
3158 if (ln2range == 64)
3159 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3160 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3161 if (ln2range == 64)
3162 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3163 pm->pm_reg + 4, 4) << 32;
3164 }
3165
3166 struct pci_map *
pci_find_bar(device_t dev,int reg)3167 pci_find_bar(device_t dev, int reg)
3168 {
3169 struct pci_devinfo *dinfo;
3170 struct pci_map *pm;
3171
3172 dinfo = device_get_ivars(dev);
3173 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3174 if (pm->pm_reg == reg)
3175 return (pm);
3176 }
3177 return (NULL);
3178 }
3179
3180 struct pci_map *
pci_first_bar(device_t dev)3181 pci_first_bar(device_t dev)
3182 {
3183 struct pci_devinfo *dinfo;
3184
3185 dinfo = device_get_ivars(dev);
3186 return (STAILQ_FIRST(&dinfo->cfg.maps));
3187 }
3188
3189 struct pci_map *
pci_next_bar(struct pci_map * pm)3190 pci_next_bar(struct pci_map *pm)
3191 {
3192 return (STAILQ_NEXT(pm, pm_link));
3193 }
3194
3195 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3196 pci_bar_enabled(device_t dev, struct pci_map *pm)
3197 {
3198 struct pci_devinfo *dinfo;
3199 uint16_t cmd;
3200
3201 dinfo = device_get_ivars(dev);
3202 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3203 !(pm->pm_value & PCIM_BIOS_ENABLE))
3204 return (0);
3205 #ifdef PCI_IOV
3206 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3207 struct pcicfg_iov *iov;
3208
3209 iov = dinfo->cfg.iov;
3210 cmd = pci_read_config(iov->iov_pf,
3211 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3212 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3213 }
3214 #endif
3215 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3216 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3217 return ((cmd & PCIM_CMD_MEMEN) != 0);
3218 else
3219 return ((cmd & PCIM_CMD_PORTEN) != 0);
3220 }
3221
3222 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3223 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3224 {
3225 struct pci_devinfo *dinfo;
3226 struct pci_map *pm, *prev;
3227
3228 dinfo = device_get_ivars(dev);
3229 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3230 pm->pm_reg = reg;
3231 pm->pm_value = value;
3232 pm->pm_size = size;
3233 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3234 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3235 reg));
3236 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3237 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3238 break;
3239 }
3240 if (prev != NULL)
3241 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3242 else
3243 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3244 return (pm);
3245 }
3246
3247 static void
pci_restore_bars(device_t dev)3248 pci_restore_bars(device_t dev)
3249 {
3250 struct pci_devinfo *dinfo;
3251 struct pci_map *pm;
3252 int ln2range;
3253
3254 dinfo = device_get_ivars(dev);
3255 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3256 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3257 ln2range = 32;
3258 else
3259 ln2range = pci_maprange(pm->pm_value);
3260 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3261 if (ln2range == 64)
3262 pci_write_config(dev, pm->pm_reg + 4,
3263 pm->pm_value >> 32, 4);
3264 }
3265 }
3266
3267 /*
3268 * Add a resource based on a pci map register. Return 1 if the map
3269 * register is a 32bit map register or 2 if it is a 64bit register.
3270 */
3271 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3272 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3273 int force, int prefetch)
3274 {
3275 struct pci_map *pm;
3276 pci_addr_t base, map, testval;
3277 pci_addr_t start, end, count;
3278 int barlen, basezero, flags, maprange, mapsize, type;
3279 uint16_t cmd;
3280 struct resource *res;
3281
3282 /*
3283 * The BAR may already exist if the device is a CardBus card
3284 * whose CIS is stored in this BAR.
3285 */
3286 pm = pci_find_bar(dev, reg);
3287 if (pm != NULL) {
3288 maprange = pci_maprange(pm->pm_value);
3289 barlen = maprange == 64 ? 2 : 1;
3290 return (barlen);
3291 }
3292
3293 pci_read_bar(dev, reg, &map, &testval, NULL);
3294 if (PCI_BAR_MEM(map)) {
3295 type = SYS_RES_MEMORY;
3296 if (map & PCIM_BAR_MEM_PREFETCH)
3297 prefetch = 1;
3298 } else
3299 type = SYS_RES_IOPORT;
3300 mapsize = pci_mapsize(testval);
3301 base = pci_mapbase(map);
3302 #ifdef __PCI_BAR_ZERO_VALID
3303 basezero = 0;
3304 #else
3305 basezero = base == 0;
3306 #endif
3307 maprange = pci_maprange(map);
3308 barlen = maprange == 64 ? 2 : 1;
3309
3310 /*
3311 * For I/O registers, if bottom bit is set, and the next bit up
3312 * isn't clear, we know we have a BAR that doesn't conform to the
3313 * spec, so ignore it. Also, sanity check the size of the data
3314 * areas to the type of memory involved. Memory must be at least
3315 * 16 bytes in size, while I/O ranges must be at least 4.
3316 */
3317 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3318 return (barlen);
3319 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3320 (type == SYS_RES_IOPORT && mapsize < 2))
3321 return (barlen);
3322
3323 /* Save a record of this BAR. */
3324 pm = pci_add_bar(dev, reg, map, mapsize);
3325 if (bootverbose) {
3326 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3327 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3328 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3329 printf(", port disabled\n");
3330 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3331 printf(", memory disabled\n");
3332 else
3333 printf(", enabled\n");
3334 }
3335
3336 /*
3337 * If base is 0, then we have problems if this architecture does
3338 * not allow that. It is best to ignore such entries for the
3339 * moment. These will be allocated later if the driver specifically
3340 * requests them. However, some removable buses look better when
3341 * all resources are allocated, so allow '0' to be overridden.
3342 *
3343 * Similarly treat maps whose values is the same as the test value
3344 * read back. These maps have had all f's written to them by the
3345 * BIOS in an attempt to disable the resources.
3346 */
3347 if (!force && (basezero || map == testval))
3348 return (barlen);
3349 if ((u_long)base != base) {
3350 device_printf(bus,
3351 "pci%d:%d:%d:%d bar %#x too many address bits",
3352 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3353 pci_get_function(dev), reg);
3354 return (barlen);
3355 }
3356
3357 /*
3358 * This code theoretically does the right thing, but has
3359 * undesirable side effects in some cases where peripherals
3360 * respond oddly to having these bits enabled. Let the user
3361 * be able to turn them off (since pci_enable_io_modes is 1 by
3362 * default).
3363 */
3364 if (pci_enable_io_modes) {
3365 /* Turn on resources that have been left off by a lazy BIOS */
3366 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3367 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3368 cmd |= PCIM_CMD_PORTEN;
3369 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3370 }
3371 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3372 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3373 cmd |= PCIM_CMD_MEMEN;
3374 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3375 }
3376 } else {
3377 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3378 return (barlen);
3379 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3380 return (barlen);
3381 }
3382
3383 count = (pci_addr_t)1 << mapsize;
3384 flags = RF_ALIGNMENT_LOG2(mapsize);
3385 if (prefetch)
3386 flags |= RF_PREFETCHABLE;
3387 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3388 start = 0; /* Let the parent decide. */
3389 end = ~0;
3390 } else {
3391 start = base;
3392 end = base + count - 1;
3393 }
3394 resource_list_add(rl, type, reg, start, end, count);
3395
3396 /*
3397 * Try to allocate the resource for this BAR from our parent
3398 * so that this resource range is already reserved. The
3399 * driver for this device will later inherit this resource in
3400 * pci_alloc_resource().
3401 */
3402 res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count,
3403 flags);
3404 if ((pci_do_realloc_bars
3405 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3406 && res == NULL && (start != 0 || end != ~0)) {
3407 /*
3408 * If the allocation fails, try to allocate a resource for
3409 * this BAR using any available range. The firmware felt
3410 * it was important enough to assign a resource, so don't
3411 * disable decoding if we can help it.
3412 */
3413 resource_list_delete(rl, type, reg);
3414 resource_list_add(rl, type, reg, 0, ~0, count);
3415 res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0,
3416 count, flags);
3417 }
3418 if (res == NULL) {
3419 /*
3420 * If the allocation fails, delete the resource list entry
3421 * and disable decoding for this device.
3422 *
3423 * If the driver requests this resource in the future,
3424 * pci_reserve_map() will try to allocate a fresh
3425 * resource range.
3426 */
3427 resource_list_delete(rl, type, reg);
3428 pci_disable_io(dev, type);
3429 if (bootverbose)
3430 device_printf(bus,
3431 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3432 pci_get_domain(dev), pci_get_bus(dev),
3433 pci_get_slot(dev), pci_get_function(dev), reg);
3434 } else {
3435 start = rman_get_start(res);
3436 pci_write_bar(dev, pm, start);
3437 }
3438 return (barlen);
3439 }
3440
3441 /*
3442 * For ATA devices we need to decide early what addressing mode to use.
3443 * Legacy demands that the primary and secondary ATA ports sits on the
3444 * same addresses that old ISA hardware did. This dictates that we use
3445 * those addresses and ignore the BAR's if we cannot set PCI native
3446 * addressing mode.
3447 */
3448 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3449 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3450 uint32_t prefetchmask)
3451 {
3452 int rid, type, progif;
3453 #if 0
3454 /* if this device supports PCI native addressing use it */
3455 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3456 if ((progif & 0x8a) == 0x8a) {
3457 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3458 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3459 printf("Trying ATA native PCI addressing mode\n");
3460 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3461 }
3462 }
3463 #endif
3464 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3465 type = SYS_RES_IOPORT;
3466 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3467 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3468 prefetchmask & (1 << 0));
3469 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3470 prefetchmask & (1 << 1));
3471 } else {
3472 rid = PCIR_BAR(0);
3473 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3474 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
3475 0x1f7, 8, 0);
3476 rid = PCIR_BAR(1);
3477 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3478 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
3479 0x3f6, 1, 0);
3480 }
3481 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3482 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3483 prefetchmask & (1 << 2));
3484 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3485 prefetchmask & (1 << 3));
3486 } else {
3487 rid = PCIR_BAR(2);
3488 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3489 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
3490 0x177, 8, 0);
3491 rid = PCIR_BAR(3);
3492 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3493 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
3494 0x376, 1, 0);
3495 }
3496 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3497 prefetchmask & (1 << 4));
3498 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3499 prefetchmask & (1 << 5));
3500 }
3501
3502 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3503 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3504 {
3505 struct pci_devinfo *dinfo = device_get_ivars(dev);
3506 pcicfgregs *cfg = &dinfo->cfg;
3507 char tunable_name[64];
3508 int irq;
3509
3510 /* Has to have an intpin to have an interrupt. */
3511 if (cfg->intpin == 0)
3512 return;
3513
3514 /* Let the user override the IRQ with a tunable. */
3515 irq = PCI_INVALID_IRQ;
3516 snprintf(tunable_name, sizeof(tunable_name),
3517 "hw.pci%d.%d.%d.INT%c.irq",
3518 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3519 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3520 irq = PCI_INVALID_IRQ;
3521
3522 /*
3523 * If we didn't get an IRQ via the tunable, then we either use the
3524 * IRQ value in the intline register or we ask the bus to route an
3525 * interrupt for us. If force_route is true, then we only use the
3526 * value in the intline register if the bus was unable to assign an
3527 * IRQ.
3528 */
3529 if (!PCI_INTERRUPT_VALID(irq)) {
3530 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3531 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3532 if (!PCI_INTERRUPT_VALID(irq))
3533 irq = cfg->intline;
3534 }
3535
3536 /* If after all that we don't have an IRQ, just bail. */
3537 if (!PCI_INTERRUPT_VALID(irq))
3538 return;
3539
3540 /* Update the config register if it changed. */
3541 if (irq != cfg->intline) {
3542 cfg->intline = irq;
3543 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3544 }
3545
3546 /* Add this IRQ as rid 0 interrupt resource. */
3547 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3548 }
3549
3550 /* Perform early OHCI takeover from SMM. */
3551 static void
ohci_early_takeover(device_t self)3552 ohci_early_takeover(device_t self)
3553 {
3554 struct resource *res;
3555 uint32_t ctl;
3556 int rid;
3557 int i;
3558
3559 rid = PCIR_BAR(0);
3560 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3561 if (res == NULL)
3562 return;
3563
3564 ctl = bus_read_4(res, OHCI_CONTROL);
3565 if (ctl & OHCI_IR) {
3566 if (bootverbose)
3567 printf("ohci early: "
3568 "SMM active, request owner change\n");
3569 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3570 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3571 DELAY(1000);
3572 ctl = bus_read_4(res, OHCI_CONTROL);
3573 }
3574 if (ctl & OHCI_IR) {
3575 if (bootverbose)
3576 printf("ohci early: "
3577 "SMM does not respond, resetting\n");
3578 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3579 }
3580 /* Disable interrupts */
3581 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3582 }
3583
3584 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3585 }
3586
3587 /* Perform early UHCI takeover from SMM. */
3588 static void
uhci_early_takeover(device_t self)3589 uhci_early_takeover(device_t self)
3590 {
3591 struct resource *res;
3592 int rid;
3593
3594 /*
3595 * Set the PIRQD enable bit and switch off all the others. We don't
3596 * want legacy support to interfere with us XXX Does this also mean
3597 * that the BIOS won't touch the keyboard anymore if it is connected
3598 * to the ports of the root hub?
3599 */
3600 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3601
3602 /* Disable interrupts */
3603 rid = PCI_UHCI_BASE_REG;
3604 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3605 if (res != NULL) {
3606 bus_write_2(res, UHCI_INTR, 0);
3607 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3608 }
3609 }
3610
3611 /* Perform early EHCI takeover from SMM. */
3612 static void
ehci_early_takeover(device_t self)3613 ehci_early_takeover(device_t self)
3614 {
3615 struct resource *res;
3616 uint32_t cparams;
3617 uint32_t eec;
3618 uint8_t eecp;
3619 uint8_t bios_sem;
3620 uint8_t offs;
3621 int rid;
3622 int i;
3623
3624 rid = PCIR_BAR(0);
3625 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3626 if (res == NULL)
3627 return;
3628
3629 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3630
3631 /* Synchronise with the BIOS if it owns the controller. */
3632 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3633 eecp = EHCI_EECP_NEXT(eec)) {
3634 eec = pci_read_config(self, eecp, 4);
3635 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3636 continue;
3637 }
3638 bios_sem = pci_read_config(self, eecp +
3639 EHCI_LEGSUP_BIOS_SEM, 1);
3640 if (bios_sem == 0) {
3641 continue;
3642 }
3643 if (bootverbose)
3644 printf("ehci early: "
3645 "SMM active, request owner change\n");
3646
3647 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3648
3649 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3650 DELAY(1000);
3651 bios_sem = pci_read_config(self, eecp +
3652 EHCI_LEGSUP_BIOS_SEM, 1);
3653 }
3654
3655 if (bios_sem != 0) {
3656 if (bootverbose)
3657 printf("ehci early: "
3658 "SMM does not respond\n");
3659 }
3660 /* Disable interrupts */
3661 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3662 bus_write_4(res, offs + EHCI_USBINTR, 0);
3663 }
3664 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3665 }
3666
3667 /* Perform early XHCI takeover from SMM. */
3668 static void
xhci_early_takeover(device_t self)3669 xhci_early_takeover(device_t self)
3670 {
3671 struct resource *res;
3672 uint32_t cparams;
3673 uint32_t eec;
3674 uint8_t eecp;
3675 uint8_t bios_sem;
3676 uint8_t offs;
3677 int rid;
3678 int i;
3679
3680 rid = PCIR_BAR(0);
3681 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3682 if (res == NULL)
3683 return;
3684
3685 cparams = bus_read_4(res, XHCI_HCSPARAMS0);
3686
3687 eec = -1;
3688
3689 /* Synchronise with the BIOS if it owns the controller. */
3690 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3691 eecp += XHCI_XECP_NEXT(eec) << 2) {
3692 eec = bus_read_4(res, eecp);
3693
3694 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3695 continue;
3696
3697 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3698 if (bios_sem == 0)
3699 continue;
3700
3701 if (bootverbose)
3702 printf("xhci early: "
3703 "SMM active, request owner change\n");
3704
3705 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3706
3707 /* wait a maximum of 5 second */
3708
3709 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3710 DELAY(1000);
3711 bios_sem = bus_read_1(res, eecp +
3712 XHCI_XECP_BIOS_SEM);
3713 }
3714
3715 if (bios_sem != 0) {
3716 if (bootverbose)
3717 printf("xhci early: "
3718 "SMM does not respond\n");
3719 }
3720
3721 /* Disable interrupts */
3722 offs = bus_read_1(res, XHCI_CAPLENGTH);
3723 bus_write_4(res, offs + XHCI_USBCMD, 0);
3724 bus_read_4(res, offs + XHCI_USBSTS);
3725 }
3726 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3727 }
3728
3729 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
3730 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3731 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3732 struct resource_list *rl)
3733 {
3734 struct resource *res;
3735 char *cp;
3736 rman_res_t start, end, count;
3737 int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3738
3739 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3740 case PCIM_HDRTYPE_BRIDGE:
3741 sec_reg = PCIR_SECBUS_1;
3742 sub_reg = PCIR_SUBBUS_1;
3743 break;
3744 case PCIM_HDRTYPE_CARDBUS:
3745 sec_reg = PCIR_SECBUS_2;
3746 sub_reg = PCIR_SUBBUS_2;
3747 break;
3748 default:
3749 return;
3750 }
3751
3752 /*
3753 * If the existing bus range is valid, attempt to reserve it
3754 * from our parent. If this fails for any reason, clear the
3755 * secbus and subbus registers.
3756 *
3757 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3758 * This would at least preserve the existing sec_bus if it is
3759 * valid.
3760 */
3761 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3762 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3763
3764 /* Quirk handling. */
3765 switch (pci_get_devid(dev)) {
3766 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3767 sup_bus = pci_read_config(dev, 0x41, 1);
3768 if (sup_bus != 0xff) {
3769 sec_bus = sup_bus + 1;
3770 sub_bus = sup_bus + 1;
3771 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3772 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3773 }
3774 break;
3775
3776 case 0x00dd10de:
3777 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3778 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3779 break;
3780 if (strncmp(cp, "Compal", 6) != 0) {
3781 freeenv(cp);
3782 break;
3783 }
3784 freeenv(cp);
3785 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3786 break;
3787 if (strncmp(cp, "08A0", 4) != 0) {
3788 freeenv(cp);
3789 break;
3790 }
3791 freeenv(cp);
3792 if (sub_bus < 0xa) {
3793 sub_bus = 0xa;
3794 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3795 }
3796 break;
3797 }
3798
3799 if (bootverbose)
3800 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3801 if (sec_bus > 0 && sub_bus >= sec_bus) {
3802 start = sec_bus;
3803 end = sub_bus;
3804 count = end - start + 1;
3805
3806 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3807
3808 /*
3809 * If requested, clear secondary bus registers in
3810 * bridge devices to force a complete renumbering
3811 * rather than reserving the existing range. However,
3812 * preserve the existing size.
3813 */
3814 if (pci_clear_buses)
3815 goto clear;
3816
3817 rid = 0;
3818 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid,
3819 start, end, count, 0);
3820 if (res != NULL)
3821 return;
3822
3823 if (bootverbose)
3824 device_printf(bus,
3825 "pci%d:%d:%d:%d secbus failed to allocate\n",
3826 pci_get_domain(dev), pci_get_bus(dev),
3827 pci_get_slot(dev), pci_get_function(dev));
3828 }
3829
3830 clear:
3831 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3832 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3833 }
3834
3835 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3836 pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start,
3837 rman_res_t end, rman_res_t count, u_int flags)
3838 {
3839 struct pci_devinfo *dinfo;
3840 pcicfgregs *cfg;
3841 struct resource_list *rl;
3842 struct resource *res;
3843 int sec_reg, sub_reg;
3844
3845 dinfo = device_get_ivars(child);
3846 cfg = &dinfo->cfg;
3847 rl = &dinfo->resources;
3848 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3849 case PCIM_HDRTYPE_BRIDGE:
3850 sec_reg = PCIR_SECBUS_1;
3851 sub_reg = PCIR_SUBBUS_1;
3852 break;
3853 case PCIM_HDRTYPE_CARDBUS:
3854 sec_reg = PCIR_SECBUS_2;
3855 sub_reg = PCIR_SUBBUS_2;
3856 break;
3857 default:
3858 return (NULL);
3859 }
3860
3861 if (*rid != 0)
3862 return (NULL);
3863
3864 if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL)
3865 resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count);
3866 if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) {
3867 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3868 start, end, count, flags & ~RF_ACTIVE);
3869 if (res == NULL) {
3870 resource_list_delete(rl, PCI_RES_BUS, *rid);
3871 device_printf(child, "allocating %ju bus%s failed\n",
3872 count, count == 1 ? "" : "es");
3873 return (NULL);
3874 }
3875 if (bootverbose)
3876 device_printf(child,
3877 "Lazy allocation of %ju bus%s at %ju\n", count,
3878 count == 1 ? "" : "es", rman_get_start(res));
3879 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3880 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3881 }
3882 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3883 end, count, flags));
3884 }
3885 #endif
3886
3887 static int
pci_ea_bei_to_rid(device_t dev,int bei)3888 pci_ea_bei_to_rid(device_t dev, int bei)
3889 {
3890 #ifdef PCI_IOV
3891 struct pci_devinfo *dinfo;
3892 int iov_pos;
3893 struct pcicfg_iov *iov;
3894
3895 dinfo = device_get_ivars(dev);
3896 iov = dinfo->cfg.iov;
3897 if (iov != NULL)
3898 iov_pos = iov->iov_pos;
3899 else
3900 iov_pos = 0;
3901 #endif
3902
3903 /* Check if matches BAR */
3904 if ((bei >= PCIM_EA_BEI_BAR_0) &&
3905 (bei <= PCIM_EA_BEI_BAR_5))
3906 return (PCIR_BAR(bei));
3907
3908 /* Check ROM */
3909 if (bei == PCIM_EA_BEI_ROM)
3910 return (PCIR_BIOS);
3911
3912 #ifdef PCI_IOV
3913 /* Check if matches VF_BAR */
3914 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
3915 (bei <= PCIM_EA_BEI_VF_BAR_5))
3916 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
3917 iov_pos);
3918 #endif
3919
3920 return (-1);
3921 }
3922
3923 int
pci_ea_is_enabled(device_t dev,int rid)3924 pci_ea_is_enabled(device_t dev, int rid)
3925 {
3926 struct pci_ea_entry *ea;
3927 struct pci_devinfo *dinfo;
3928
3929 dinfo = device_get_ivars(dev);
3930
3931 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3932 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
3933 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
3934 }
3935
3936 return (0);
3937 }
3938
3939 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)3940 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
3941 {
3942 struct pci_ea_entry *ea;
3943 struct pci_devinfo *dinfo;
3944 pci_addr_t start, end, count;
3945 struct resource_list *rl;
3946 int type, flags, rid;
3947 struct resource *res;
3948 uint32_t tmp;
3949 #ifdef PCI_IOV
3950 struct pcicfg_iov *iov;
3951 #endif
3952
3953 dinfo = device_get_ivars(dev);
3954 rl = &dinfo->resources;
3955 flags = 0;
3956
3957 #ifdef PCI_IOV
3958 iov = dinfo->cfg.iov;
3959 #endif
3960
3961 if (dinfo->cfg.ea.ea_location == 0)
3962 return;
3963
3964 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3965 /*
3966 * TODO: Ignore EA-BAR if is not enabled.
3967 * Currently the EA implementation supports
3968 * only situation, where EA structure contains
3969 * predefined entries. In case they are not enabled
3970 * leave them unallocated and proceed with
3971 * a legacy-BAR mechanism.
3972 */
3973 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
3974 continue;
3975
3976 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
3977 case PCIM_EA_P_MEM_PREFETCH:
3978 case PCIM_EA_P_VF_MEM_PREFETCH:
3979 flags = RF_PREFETCHABLE;
3980 /* FALLTHROUGH */
3981 case PCIM_EA_P_VF_MEM:
3982 case PCIM_EA_P_MEM:
3983 type = SYS_RES_MEMORY;
3984 break;
3985 case PCIM_EA_P_IO:
3986 type = SYS_RES_IOPORT;
3987 break;
3988 default:
3989 continue;
3990 }
3991
3992 if (alloc_iov != 0) {
3993 #ifdef PCI_IOV
3994 /* Allocating IOV, confirm BEI matches */
3995 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
3996 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
3997 continue;
3998 #else
3999 continue;
4000 #endif
4001 } else {
4002 /* Allocating BAR, confirm BEI matches */
4003 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
4004 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
4005 (ea->eae_bei != PCIM_EA_BEI_ROM))
4006 continue;
4007 }
4008
4009 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
4010 if (rid < 0)
4011 continue;
4012
4013 /* Skip resources already allocated by EA */
4014 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
4015 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
4016 continue;
4017
4018 start = ea->eae_base;
4019 count = ea->eae_max_offset + 1;
4020 #ifdef PCI_IOV
4021 if (iov != NULL)
4022 count = count * iov->iov_num_vfs;
4023 #endif
4024 end = start + count - 1;
4025 if (count == 0)
4026 continue;
4027
4028 resource_list_add(rl, type, rid, start, end, count);
4029 res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count,
4030 flags);
4031 if (res == NULL) {
4032 resource_list_delete(rl, type, rid);
4033
4034 /*
4035 * Failed to allocate using EA, disable entry.
4036 * Another attempt to allocation will be performed
4037 * further, but this time using legacy BAR registers
4038 */
4039 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4040 tmp &= ~PCIM_EA_ENABLE;
4041 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4042
4043 /*
4044 * Disabling entry might fail in case it is hardwired.
4045 * Read flags again to match current status.
4046 */
4047 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4048
4049 continue;
4050 }
4051
4052 /* As per specification, fill BAR with zeros */
4053 pci_write_config(dev, rid, 0, 4);
4054 }
4055 }
4056
4057 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4058 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4059 {
4060 struct pci_devinfo *dinfo;
4061 pcicfgregs *cfg;
4062 struct resource_list *rl;
4063 const struct pci_quirk *q;
4064 uint32_t devid;
4065 int i;
4066
4067 dinfo = device_get_ivars(dev);
4068 cfg = &dinfo->cfg;
4069 rl = &dinfo->resources;
4070 devid = (cfg->device << 16) | cfg->vendor;
4071
4072 /* Allocate resources using Enhanced Allocation */
4073 pci_add_resources_ea(bus, dev, 0);
4074
4075 /* ATA devices needs special map treatment */
4076 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4077 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4078 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4079 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4080 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4081 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4082 else
4083 for (i = 0; i < cfg->nummaps;) {
4084 /* Skip resources already managed by EA */
4085 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4086 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4087 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4088 i++;
4089 continue;
4090 }
4091
4092 /*
4093 * Skip quirked resources.
4094 */
4095 for (q = &pci_quirks[0]; q->devid != 0; q++)
4096 if (q->devid == devid &&
4097 q->type == PCI_QUIRK_UNMAP_REG &&
4098 q->arg1 == PCIR_BAR(i))
4099 break;
4100 if (q->devid != 0) {
4101 i++;
4102 continue;
4103 }
4104 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4105 prefetchmask & (1 << i));
4106 }
4107
4108 /*
4109 * Add additional, quirked resources.
4110 */
4111 for (q = &pci_quirks[0]; q->devid != 0; q++)
4112 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4113 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4114
4115 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
4116 #ifdef __PCI_REROUTE_INTERRUPT
4117 /*
4118 * Try to re-route interrupts. Sometimes the BIOS or
4119 * firmware may leave bogus values in these registers.
4120 * If the re-route fails, then just stick with what we
4121 * have.
4122 */
4123 pci_assign_interrupt(bus, dev, 1);
4124 #else
4125 pci_assign_interrupt(bus, dev, 0);
4126 #endif
4127 }
4128
4129 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4130 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4131 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4132 xhci_early_takeover(dev);
4133 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4134 ehci_early_takeover(dev);
4135 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4136 ohci_early_takeover(dev);
4137 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4138 uhci_early_takeover(dev);
4139 }
4140
4141 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
4142 /*
4143 * Reserve resources for secondary bus ranges behind bridge
4144 * devices.
4145 */
4146 pci_reserve_secbus(bus, dev, cfg, rl);
4147 #endif
4148 }
4149
4150 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4151 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4152 int slot, int func)
4153 {
4154 struct pci_devinfo *dinfo;
4155
4156 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4157 if (dinfo != NULL)
4158 pci_add_child(dev, dinfo);
4159
4160 return (dinfo);
4161 }
4162
4163 void
pci_add_children(device_t dev,int domain,int busno)4164 pci_add_children(device_t dev, int domain, int busno)
4165 {
4166 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4167 device_t pcib = device_get_parent(dev);
4168 struct pci_devinfo *dinfo;
4169 int maxslots;
4170 int s, f, pcifunchigh;
4171 uint8_t hdrtype;
4172 int first_func;
4173
4174 /*
4175 * Try to detect a device at slot 0, function 0. If it exists, try to
4176 * enable ARI. We must enable ARI before detecting the rest of the
4177 * functions on this bus as ARI changes the set of slots and functions
4178 * that are legal on this bus.
4179 */
4180 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4181 if (dinfo != NULL && pci_enable_ari)
4182 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4183
4184 /*
4185 * Start looking for new devices on slot 0 at function 1 because we
4186 * just identified the device at slot 0, function 0.
4187 */
4188 first_func = 1;
4189
4190 maxslots = PCIB_MAXSLOTS(pcib);
4191 for (s = 0; s <= maxslots; s++, first_func = 0) {
4192 pcifunchigh = 0;
4193 f = 0;
4194 DELAY(1);
4195
4196 /* If function 0 is not present, skip to the next slot. */
4197 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4198 continue;
4199 hdrtype = REG(PCIR_HDRTYPE, 1);
4200 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4201 continue;
4202 if (hdrtype & PCIM_MFDEV)
4203 pcifunchigh = PCIB_MAXFUNCS(pcib);
4204 for (f = first_func; f <= pcifunchigh; f++)
4205 pci_identify_function(pcib, dev, domain, busno, s, f);
4206 }
4207 #undef REG
4208 }
4209
4210 int
pci_rescan_method(device_t dev)4211 pci_rescan_method(device_t dev)
4212 {
4213 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4214 device_t pcib = device_get_parent(dev);
4215 device_t child, *devlist, *unchanged;
4216 int devcount, error, i, j, maxslots, oldcount;
4217 int busno, domain, s, f, pcifunchigh;
4218 uint8_t hdrtype;
4219
4220 /* No need to check for ARI on a rescan. */
4221 error = device_get_children(dev, &devlist, &devcount);
4222 if (error)
4223 return (error);
4224 if (devcount != 0) {
4225 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4226 M_NOWAIT | M_ZERO);
4227 if (unchanged == NULL) {
4228 free(devlist, M_TEMP);
4229 return (ENOMEM);
4230 }
4231 } else
4232 unchanged = NULL;
4233
4234 domain = pcib_get_domain(dev);
4235 busno = pcib_get_bus(dev);
4236 maxslots = PCIB_MAXSLOTS(pcib);
4237 for (s = 0; s <= maxslots; s++) {
4238 /* If function 0 is not present, skip to the next slot. */
4239 f = 0;
4240 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4241 continue;
4242 pcifunchigh = 0;
4243 hdrtype = REG(PCIR_HDRTYPE, 1);
4244 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4245 continue;
4246 if (hdrtype & PCIM_MFDEV)
4247 pcifunchigh = PCIB_MAXFUNCS(pcib);
4248 for (f = 0; f <= pcifunchigh; f++) {
4249 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4250 continue;
4251
4252 /*
4253 * Found a valid function. Check if a
4254 * device_t for this device already exists.
4255 */
4256 for (i = 0; i < devcount; i++) {
4257 child = devlist[i];
4258 if (child == NULL)
4259 continue;
4260 if (pci_get_slot(child) == s &&
4261 pci_get_function(child) == f) {
4262 unchanged[i] = child;
4263 goto next_func;
4264 }
4265 }
4266
4267 pci_identify_function(pcib, dev, domain, busno, s, f);
4268 next_func:;
4269 }
4270 }
4271
4272 /* Remove devices that are no longer present. */
4273 for (i = 0; i < devcount; i++) {
4274 if (unchanged[i] != NULL)
4275 continue;
4276 device_delete_child(dev, devlist[i]);
4277 }
4278
4279 free(devlist, M_TEMP);
4280 oldcount = devcount;
4281
4282 /* Try to attach the devices just added. */
4283 error = device_get_children(dev, &devlist, &devcount);
4284 if (error) {
4285 free(unchanged, M_TEMP);
4286 return (error);
4287 }
4288
4289 for (i = 0; i < devcount; i++) {
4290 for (j = 0; j < oldcount; j++) {
4291 if (devlist[i] == unchanged[j])
4292 goto next_device;
4293 }
4294
4295 device_probe_and_attach(devlist[i]);
4296 next_device:;
4297 }
4298
4299 free(unchanged, M_TEMP);
4300 free(devlist, M_TEMP);
4301 return (0);
4302 #undef REG
4303 }
4304
4305 #ifdef PCI_IOV
4306 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4307 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4308 uint16_t did)
4309 {
4310 struct pci_devinfo *vf_dinfo;
4311 device_t pcib;
4312 int busno, slot, func;
4313
4314 pcib = device_get_parent(bus);
4315
4316 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4317
4318 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4319 slot, func, vid, did);
4320
4321 vf_dinfo->cfg.flags |= PCICFG_VF;
4322 pci_add_child(bus, vf_dinfo);
4323
4324 return (vf_dinfo->cfg.dev);
4325 }
4326
4327 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4328 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4329 uint16_t vid, uint16_t did)
4330 {
4331
4332 return (pci_add_iov_child(bus, pf, rid, vid, did));
4333 }
4334 #endif
4335
4336 /*
4337 * For PCIe device set Max_Payload_Size to match PCIe root's.
4338 */
4339 static void
pcie_setup_mps(device_t dev)4340 pcie_setup_mps(device_t dev)
4341 {
4342 struct pci_devinfo *dinfo = device_get_ivars(dev);
4343 device_t root;
4344 uint16_t rmps, mmps, mps;
4345
4346 if (dinfo->cfg.pcie.pcie_location == 0)
4347 return;
4348 root = pci_find_pcie_root_port(dev);
4349 if (root == NULL)
4350 return;
4351 /* Check whether the MPS is already configured. */
4352 rmps = pcie_read_config(root, PCIER_DEVICE_CTL, 2) &
4353 PCIEM_CTL_MAX_PAYLOAD;
4354 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4355 PCIEM_CTL_MAX_PAYLOAD;
4356 if (mps == rmps)
4357 return;
4358 /* Check whether the device is capable of the root's MPS. */
4359 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4360 PCIEM_CAP_MAX_PAYLOAD) << 5;
4361 if (rmps > mmps) {
4362 /*
4363 * The device is unable to handle root's MPS. Limit root.
4364 * XXX: We should traverse through all the tree, applying
4365 * it to all the devices.
4366 */
4367 pcie_adjust_config(root, PCIER_DEVICE_CTL,
4368 PCIEM_CTL_MAX_PAYLOAD, mmps, 2);
4369 } else {
4370 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
4371 PCIEM_CTL_MAX_PAYLOAD, rmps, 2);
4372 }
4373 }
4374
4375 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4376 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4377 {
4378 int aer;
4379 uint32_t r;
4380 uint16_t r2;
4381
4382 if (dinfo->cfg.pcie.pcie_location != 0 &&
4383 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4384 r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4385 PCIER_ROOT_CTL, 2);
4386 r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR |
4387 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL);
4388 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4389 PCIER_ROOT_CTL, r2, 2);
4390 }
4391 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4392 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4393 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4394 if (r != 0 && bootverbose) {
4395 pci_printf(&dinfo->cfg,
4396 "clearing AER UC 0x%08x -> 0x%08x\n",
4397 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4398 4));
4399 }
4400
4401 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4402 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4403 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4404 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4405 PCIM_AER_UC_POISONED_TLP |
4406 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4407 PCIM_AER_UC_COMPLETION_TIMEOUT |
4408 PCIM_AER_UC_COMPLETER_ABORT |
4409 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4410 PCIM_AER_UC_RECEIVER_OVERFLOW |
4411 PCIM_AER_UC_MALFORMED_TLP |
4412 PCIM_AER_UC_ECRC_ERROR |
4413 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4414 PCIM_AER_UC_ACS_VIOLATION |
4415 PCIM_AER_UC_INTERNAL_ERROR |
4416 PCIM_AER_UC_MC_BLOCKED_TLP |
4417 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4418 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4419 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4420
4421 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4422 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4423 if (r != 0 && bootverbose) {
4424 pci_printf(&dinfo->cfg,
4425 "clearing AER COR 0x%08x -> 0x%08x\n",
4426 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4427 4));
4428 }
4429
4430 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4431 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4432 PCIM_AER_COR_BAD_TLP |
4433 PCIM_AER_COR_BAD_DLLP |
4434 PCIM_AER_COR_REPLAY_ROLLOVER |
4435 PCIM_AER_COR_REPLAY_TIMEOUT |
4436 PCIM_AER_COR_ADVISORY_NF_ERROR |
4437 PCIM_AER_COR_INTERNAL_ERROR |
4438 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4439 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4440
4441 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4442 PCIER_DEVICE_CTL, 2);
4443 r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4444 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4445 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4446 PCIER_DEVICE_CTL, r, 2);
4447 }
4448 }
4449
4450 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4451 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4452 {
4453 device_t dev;
4454
4455 dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1);
4456 device_set_ivars(dev, dinfo);
4457 resource_list_init(&dinfo->resources);
4458 pci_cfg_save(dev, dinfo, 0);
4459 pci_cfg_restore(dev, dinfo);
4460 pci_print_verbose(dinfo);
4461 pci_add_resources(bus, dev, 0, 0);
4462 if (pci_enable_mps_tune)
4463 pcie_setup_mps(dev);
4464 pci_child_added(dinfo->cfg.dev);
4465
4466 if (pci_clear_aer_on_attach)
4467 pci_add_child_clear_aer(dev, dinfo);
4468
4469 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4470 }
4471
4472 void
pci_child_added_method(device_t dev,device_t child)4473 pci_child_added_method(device_t dev, device_t child)
4474 {
4475
4476 }
4477
4478 static int
pci_probe(device_t dev)4479 pci_probe(device_t dev)
4480 {
4481
4482 device_set_desc(dev, "PCI bus");
4483
4484 /* Allow other subclasses to override this driver. */
4485 return (BUS_PROBE_GENERIC);
4486 }
4487
4488 int
pci_attach_common(device_t dev)4489 pci_attach_common(device_t dev)
4490 {
4491 struct pci_softc *sc;
4492 int busno, domain;
4493 #ifdef PCI_RES_BUS
4494 int rid;
4495 #endif
4496
4497 sc = device_get_softc(dev);
4498 domain = pcib_get_domain(dev);
4499 busno = pcib_get_bus(dev);
4500 #ifdef PCI_RES_BUS
4501 rid = 0;
4502 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4503 1, 0);
4504 if (sc->sc_bus == NULL) {
4505 device_printf(dev, "failed to allocate bus number\n");
4506 return (ENXIO);
4507 }
4508 #endif
4509 if (bootverbose)
4510 device_printf(dev, "domain=%d, physical bus=%d\n",
4511 domain, busno);
4512 sc->sc_dma_tag = bus_get_dma_tag(dev);
4513 return (0);
4514 }
4515
4516 int
pci_attach(device_t dev)4517 pci_attach(device_t dev)
4518 {
4519 int busno, domain, error;
4520
4521 error = pci_attach_common(dev);
4522 if (error)
4523 return (error);
4524
4525 /*
4526 * Since there can be multiple independently numbered PCI
4527 * buses on systems with multiple PCI domains, we can't use
4528 * the unit number to decide which bus we are probing. We ask
4529 * the parent pcib what our domain and bus numbers are.
4530 */
4531 domain = pcib_get_domain(dev);
4532 busno = pcib_get_bus(dev);
4533 pci_add_children(dev, domain, busno);
4534 return (bus_generic_attach(dev));
4535 }
4536
4537 int
pci_detach(device_t dev)4538 pci_detach(device_t dev)
4539 {
4540 #ifdef PCI_RES_BUS
4541 struct pci_softc *sc;
4542 #endif
4543 int error;
4544
4545 error = bus_generic_detach(dev);
4546 if (error)
4547 return (error);
4548 #ifdef PCI_RES_BUS
4549 sc = device_get_softc(dev);
4550 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
4551 if (error)
4552 return (error);
4553 #endif
4554 return (device_delete_children(dev));
4555 }
4556
4557 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)4558 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
4559 {
4560 int line, unit;
4561 const char *at;
4562 char me1[24], me2[32];
4563 uint8_t b, s, f;
4564 uint32_t d;
4565
4566 d = pci_get_domain(child);
4567 b = pci_get_bus(child);
4568 s = pci_get_slot(child);
4569 f = pci_get_function(child);
4570 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
4571 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
4572 line = 0;
4573 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
4574 resource_string_value(name, unit, "at", &at);
4575 if (strcmp(at, me1) != 0 && strcmp(at, me2) != 0)
4576 continue; /* No match, try next candidate */
4577 *unitp = unit;
4578 return;
4579 }
4580 }
4581
4582 static void
pci_set_power_child(device_t dev,device_t child,int state)4583 pci_set_power_child(device_t dev, device_t child, int state)
4584 {
4585 device_t pcib;
4586 int dstate;
4587
4588 /*
4589 * Set the device to the given state. If the firmware suggests
4590 * a different power state, use it instead. If power management
4591 * is not present, the firmware is responsible for managing
4592 * device power. Skip children who aren't attached since they
4593 * are handled separately.
4594 */
4595 pcib = device_get_parent(dev);
4596 dstate = state;
4597 if (device_is_attached(child) &&
4598 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
4599 pci_set_powerstate(child, dstate);
4600 }
4601
4602 int
pci_suspend_child(device_t dev,device_t child)4603 pci_suspend_child(device_t dev, device_t child)
4604 {
4605 struct pci_devinfo *dinfo;
4606 struct resource_list_entry *rle;
4607 int error;
4608
4609 dinfo = device_get_ivars(child);
4610
4611 /*
4612 * Save the PCI configuration space for the child and set the
4613 * device in the appropriate power state for this sleep state.
4614 */
4615 pci_cfg_save(child, dinfo, 0);
4616
4617 /* Suspend devices before potentially powering them down. */
4618 error = bus_generic_suspend_child(dev, child);
4619
4620 if (error)
4621 return (error);
4622
4623 if (pci_do_power_suspend) {
4624 /*
4625 * Make sure this device's interrupt handler is not invoked
4626 * in the case the device uses a shared interrupt that can
4627 * be raised by some other device.
4628 * This is applicable only to regular (legacy) PCI interrupts
4629 * as MSI/MSI-X interrupts are never shared.
4630 */
4631 rle = resource_list_find(&dinfo->resources,
4632 SYS_RES_IRQ, 0);
4633 if (rle != NULL && rle->res != NULL)
4634 (void)bus_suspend_intr(child, rle->res);
4635 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
4636 }
4637
4638 return (0);
4639 }
4640
4641 int
pci_resume_child(device_t dev,device_t child)4642 pci_resume_child(device_t dev, device_t child)
4643 {
4644 struct pci_devinfo *dinfo;
4645 struct resource_list_entry *rle;
4646
4647 if (pci_do_power_resume)
4648 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
4649
4650 dinfo = device_get_ivars(child);
4651 pci_cfg_restore(child, dinfo);
4652 if (!device_is_attached(child))
4653 pci_cfg_save(child, dinfo, 1);
4654
4655 bus_generic_resume_child(dev, child);
4656
4657 /*
4658 * Allow interrupts only after fully resuming the driver and hardware.
4659 */
4660 if (pci_do_power_suspend) {
4661 /* See pci_suspend_child for details. */
4662 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
4663 if (rle != NULL && rle->res != NULL)
4664 (void)bus_resume_intr(child, rle->res);
4665 }
4666
4667 return (0);
4668 }
4669
4670 int
pci_resume(device_t dev)4671 pci_resume(device_t dev)
4672 {
4673 device_t child, *devlist;
4674 int error, i, numdevs;
4675
4676 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
4677 return (error);
4678
4679 /*
4680 * Resume critical devices first, then everything else later.
4681 */
4682 for (i = 0; i < numdevs; i++) {
4683 child = devlist[i];
4684 switch (pci_get_class(child)) {
4685 case PCIC_DISPLAY:
4686 case PCIC_MEMORY:
4687 case PCIC_BRIDGE:
4688 case PCIC_BASEPERIPH:
4689 BUS_RESUME_CHILD(dev, child);
4690 break;
4691 }
4692 }
4693 for (i = 0; i < numdevs; i++) {
4694 child = devlist[i];
4695 switch (pci_get_class(child)) {
4696 case PCIC_DISPLAY:
4697 case PCIC_MEMORY:
4698 case PCIC_BRIDGE:
4699 case PCIC_BASEPERIPH:
4700 break;
4701 default:
4702 BUS_RESUME_CHILD(dev, child);
4703 }
4704 }
4705 free(devlist, M_TEMP);
4706 return (0);
4707 }
4708
4709 static void
pci_load_vendor_data(void)4710 pci_load_vendor_data(void)
4711 {
4712 caddr_t data;
4713 void *ptr;
4714 size_t sz;
4715
4716 data = preload_search_by_type("pci_vendor_data");
4717 if (data != NULL) {
4718 ptr = preload_fetch_addr(data);
4719 sz = preload_fetch_size(data);
4720 if (ptr != NULL && sz != 0) {
4721 pci_vendordata = ptr;
4722 pci_vendordata_size = sz;
4723 /* terminate the database */
4724 pci_vendordata[pci_vendordata_size] = '\n';
4725 }
4726 }
4727 }
4728
4729 void
pci_driver_added(device_t dev,driver_t * driver)4730 pci_driver_added(device_t dev, driver_t *driver)
4731 {
4732 int numdevs;
4733 device_t *devlist;
4734 device_t child;
4735 struct pci_devinfo *dinfo;
4736 int i;
4737
4738 if (bootverbose)
4739 device_printf(dev, "driver added\n");
4740 DEVICE_IDENTIFY(driver, dev);
4741 if (device_get_children(dev, &devlist, &numdevs) != 0)
4742 return;
4743 for (i = 0; i < numdevs; i++) {
4744 child = devlist[i];
4745 if (device_get_state(child) != DS_NOTPRESENT)
4746 continue;
4747 dinfo = device_get_ivars(child);
4748 pci_print_verbose(dinfo);
4749 if (bootverbose)
4750 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
4751 pci_cfg_restore(child, dinfo);
4752 if (device_probe_and_attach(child) != 0)
4753 pci_child_detached(dev, child);
4754 }
4755 free(devlist, M_TEMP);
4756 }
4757
4758 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)4759 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
4760 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
4761 {
4762 struct pci_devinfo *dinfo;
4763 struct msix_table_entry *mte;
4764 struct msix_vector *mv;
4765 uint64_t addr;
4766 uint32_t data;
4767 void *cookie;
4768 int error, rid;
4769
4770 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
4771 arg, &cookie);
4772 if (error)
4773 return (error);
4774
4775 /* If this is not a direct child, just bail out. */
4776 if (device_get_parent(child) != dev) {
4777 *cookiep = cookie;
4778 return(0);
4779 }
4780
4781 rid = rman_get_rid(irq);
4782 if (rid == 0) {
4783 /* Make sure that INTx is enabled */
4784 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4785 } else {
4786 /*
4787 * Check to see if the interrupt is MSI or MSI-X.
4788 * Ask our parent to map the MSI and give
4789 * us the address and data register values.
4790 * If we fail for some reason, teardown the
4791 * interrupt handler.
4792 */
4793 dinfo = device_get_ivars(child);
4794 if (dinfo->cfg.msi.msi_alloc > 0) {
4795 if (dinfo->cfg.msi.msi_addr == 0) {
4796 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
4797 ("MSI has handlers, but vectors not mapped"));
4798 error = PCIB_MAP_MSI(device_get_parent(dev),
4799 child, rman_get_start(irq), &addr, &data);
4800 if (error)
4801 goto bad;
4802 dinfo->cfg.msi.msi_addr = addr;
4803 dinfo->cfg.msi.msi_data = data;
4804 }
4805 if (dinfo->cfg.msi.msi_handlers == 0)
4806 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
4807 dinfo->cfg.msi.msi_data);
4808 dinfo->cfg.msi.msi_handlers++;
4809 } else {
4810 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4811 ("No MSI or MSI-X interrupts allocated"));
4812 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4813 ("MSI-X index too high"));
4814 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4815 KASSERT(mte->mte_vector != 0, ("no message vector"));
4816 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
4817 KASSERT(mv->mv_irq == rman_get_start(irq),
4818 ("IRQ mismatch"));
4819 if (mv->mv_address == 0) {
4820 KASSERT(mte->mte_handlers == 0,
4821 ("MSI-X table entry has handlers, but vector not mapped"));
4822 error = PCIB_MAP_MSI(device_get_parent(dev),
4823 child, rman_get_start(irq), &addr, &data);
4824 if (error)
4825 goto bad;
4826 mv->mv_address = addr;
4827 mv->mv_data = data;
4828 }
4829
4830 /*
4831 * The MSIX table entry must be made valid by
4832 * incrementing the mte_handlers before
4833 * calling pci_enable_msix() and
4834 * pci_resume_msix(). Else the MSIX rewrite
4835 * table quirk will not work as expected.
4836 */
4837 mte->mte_handlers++;
4838 if (mte->mte_handlers == 1) {
4839 pci_enable_msix(child, rid - 1, mv->mv_address,
4840 mv->mv_data);
4841 pci_unmask_msix(child, rid - 1);
4842 }
4843 }
4844
4845 /*
4846 * Make sure that INTx is disabled if we are using MSI/MSI-X,
4847 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
4848 * in which case we "enable" INTx so MSI/MSI-X actually works.
4849 */
4850 if (!pci_has_quirk(pci_get_devid(child),
4851 PCI_QUIRK_MSI_INTX_BUG))
4852 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4853 else
4854 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4855 bad:
4856 if (error) {
4857 (void)bus_generic_teardown_intr(dev, child, irq,
4858 cookie);
4859 return (error);
4860 }
4861 }
4862 *cookiep = cookie;
4863 return (0);
4864 }
4865
4866 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)4867 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
4868 void *cookie)
4869 {
4870 struct msix_table_entry *mte;
4871 struct resource_list_entry *rle;
4872 struct pci_devinfo *dinfo;
4873 int error, rid;
4874
4875 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
4876 return (EINVAL);
4877
4878 /* If this isn't a direct child, just bail out */
4879 if (device_get_parent(child) != dev)
4880 return(bus_generic_teardown_intr(dev, child, irq, cookie));
4881
4882 rid = rman_get_rid(irq);
4883 if (rid == 0) {
4884 /* Mask INTx */
4885 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4886 } else {
4887 /*
4888 * Check to see if the interrupt is MSI or MSI-X. If so,
4889 * decrement the appropriate handlers count and mask the
4890 * MSI-X message, or disable MSI messages if the count
4891 * drops to 0.
4892 */
4893 dinfo = device_get_ivars(child);
4894 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
4895 if (rle->res != irq)
4896 return (EINVAL);
4897 if (dinfo->cfg.msi.msi_alloc > 0) {
4898 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
4899 ("MSI-X index too high"));
4900 if (dinfo->cfg.msi.msi_handlers == 0)
4901 return (EINVAL);
4902 dinfo->cfg.msi.msi_handlers--;
4903 if (dinfo->cfg.msi.msi_handlers == 0)
4904 pci_disable_msi(child);
4905 } else {
4906 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4907 ("No MSI or MSI-X interrupts allocated"));
4908 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4909 ("MSI-X index too high"));
4910 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4911 if (mte->mte_handlers == 0)
4912 return (EINVAL);
4913 mte->mte_handlers--;
4914 if (mte->mte_handlers == 0)
4915 pci_mask_msix(child, rid - 1);
4916 }
4917 }
4918 error = bus_generic_teardown_intr(dev, child, irq, cookie);
4919 if (rid > 0)
4920 KASSERT(error == 0,
4921 ("%s: generic teardown failed for MSI/MSI-X", __func__));
4922 return (error);
4923 }
4924
4925 int
pci_print_child(device_t dev,device_t child)4926 pci_print_child(device_t dev, device_t child)
4927 {
4928 struct pci_devinfo *dinfo;
4929 struct resource_list *rl;
4930 int retval = 0;
4931
4932 dinfo = device_get_ivars(child);
4933 rl = &dinfo->resources;
4934
4935 retval += bus_print_child_header(dev, child);
4936
4937 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
4938 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
4939 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
4940 if (device_get_flags(dev))
4941 retval += printf(" flags %#x", device_get_flags(dev));
4942
4943 retval += printf(" at device %d.%d", pci_get_slot(child),
4944 pci_get_function(child));
4945
4946 retval += bus_print_child_domain(dev, child);
4947 retval += bus_print_child_footer(dev, child);
4948
4949 return (retval);
4950 }
4951
4952 static const struct
4953 {
4954 int class;
4955 int subclass;
4956 int report; /* 0 = bootverbose, 1 = always */
4957 const char *desc;
4958 } pci_nomatch_tab[] = {
4959 {PCIC_OLD, -1, 1, "old"},
4960 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
4961 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
4962 {PCIC_STORAGE, -1, 1, "mass storage"},
4963 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
4964 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
4965 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
4966 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
4967 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
4968 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
4969 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
4970 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
4971 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
4972 {PCIC_NETWORK, -1, 1, "network"},
4973 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
4974 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
4975 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
4976 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
4977 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
4978 {PCIC_DISPLAY, -1, 1, "display"},
4979 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
4980 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
4981 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
4982 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
4983 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
4984 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
4985 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
4986 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
4987 {PCIC_MEMORY, -1, 1, "memory"},
4988 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
4989 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
4990 {PCIC_BRIDGE, -1, 1, "bridge"},
4991 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
4992 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
4993 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
4994 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
4995 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
4996 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
4997 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
4998 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
4999 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
5000 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
5001 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
5002 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
5003 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
5004 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
5005 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
5006 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
5007 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
5008 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
5009 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
5010 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
5011 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
5012 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
5013 {PCIC_INPUTDEV, -1, 1, "input device"},
5014 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
5015 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
5016 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
5017 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
5018 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5019 {PCIC_DOCKING, -1, 1, "docking station"},
5020 {PCIC_PROCESSOR, -1, 1, "processor"},
5021 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5022 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5023 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5024 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5025 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5026 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5027 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5028 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5029 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5030 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5031 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5032 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5033 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5034 {PCIC_SATCOM, -1, 1, "satellite communication"},
5035 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5036 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5037 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5038 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5039 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5040 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5041 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5042 {PCIC_DASP, -1, 0, "dasp"},
5043 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5044 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5045 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5046 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5047 {0, 0, 0, NULL}
5048 };
5049
5050 void
pci_probe_nomatch(device_t dev,device_t child)5051 pci_probe_nomatch(device_t dev, device_t child)
5052 {
5053 int i, report;
5054 const char *cp, *scp;
5055 char *device;
5056
5057 /*
5058 * Look for a listing for this device in a loaded device database.
5059 */
5060 report = 1;
5061 if ((device = pci_describe_device(child)) != NULL) {
5062 device_printf(dev, "<%s>", device);
5063 free(device, M_DEVBUF);
5064 } else {
5065 /*
5066 * Scan the class/subclass descriptions for a general
5067 * description.
5068 */
5069 cp = "unknown";
5070 scp = NULL;
5071 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5072 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5073 if (pci_nomatch_tab[i].subclass == -1) {
5074 cp = pci_nomatch_tab[i].desc;
5075 report = pci_nomatch_tab[i].report;
5076 } else if (pci_nomatch_tab[i].subclass ==
5077 pci_get_subclass(child)) {
5078 scp = pci_nomatch_tab[i].desc;
5079 report = pci_nomatch_tab[i].report;
5080 }
5081 }
5082 }
5083 if (report || bootverbose) {
5084 device_printf(dev, "<%s%s%s>",
5085 cp ? cp : "",
5086 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5087 scp ? scp : "");
5088 }
5089 }
5090 if (report || bootverbose) {
5091 printf(" at device %d.%d (no driver attached)\n",
5092 pci_get_slot(child), pci_get_function(child));
5093 }
5094 pci_cfg_save(child, device_get_ivars(child), 1);
5095 }
5096
5097 void
pci_child_detached(device_t dev,device_t child)5098 pci_child_detached(device_t dev, device_t child)
5099 {
5100 struct pci_devinfo *dinfo;
5101 struct resource_list *rl;
5102
5103 dinfo = device_get_ivars(child);
5104 rl = &dinfo->resources;
5105
5106 /*
5107 * Have to deallocate IRQs before releasing any MSI messages and
5108 * have to release MSI messages before deallocating any memory
5109 * BARs.
5110 */
5111 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5112 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5113 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5114 if (dinfo->cfg.msi.msi_alloc != 0)
5115 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5116 "vectors\n", dinfo->cfg.msi.msi_alloc);
5117 else
5118 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5119 "vectors\n", dinfo->cfg.msix.msix_alloc);
5120 (void)pci_release_msi(child);
5121 }
5122 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5123 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5124 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5125 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5126 #ifdef PCI_RES_BUS
5127 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5128 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5129 #endif
5130
5131 pci_cfg_save(child, dinfo, 1);
5132 }
5133
5134 /*
5135 * Parse the PCI device database, if loaded, and return a pointer to a
5136 * description of the device.
5137 *
5138 * The database is flat text formatted as follows:
5139 *
5140 * Any line not in a valid format is ignored.
5141 * Lines are terminated with newline '\n' characters.
5142 *
5143 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5144 * the vendor name.
5145 *
5146 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5147 * - devices cannot be listed without a corresponding VENDOR line.
5148 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5149 * another TAB, then the device name.
5150 */
5151
5152 /*
5153 * Assuming (ptr) points to the beginning of a line in the database,
5154 * return the vendor or device and description of the next entry.
5155 * The value of (vendor) or (device) inappropriate for the entry type
5156 * is set to -1. Returns nonzero at the end of the database.
5157 *
5158 * Note that this is slightly unrobust in the face of corrupt data;
5159 * we attempt to safeguard against this by spamming the end of the
5160 * database with a newline when we initialise.
5161 */
5162 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5163 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5164 {
5165 char *cp = *ptr;
5166 int left;
5167
5168 *device = -1;
5169 *vendor = -1;
5170 **desc = '\0';
5171 for (;;) {
5172 left = pci_vendordata_size - (cp - pci_vendordata);
5173 if (left <= 0) {
5174 *ptr = cp;
5175 return(1);
5176 }
5177
5178 /* vendor entry? */
5179 if (*cp != '\t' &&
5180 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5181 break;
5182 /* device entry? */
5183 if (*cp == '\t' &&
5184 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5185 break;
5186
5187 /* skip to next line */
5188 while (*cp != '\n' && left > 0) {
5189 cp++;
5190 left--;
5191 }
5192 if (*cp == '\n') {
5193 cp++;
5194 left--;
5195 }
5196 }
5197 /* skip to next line */
5198 while (*cp != '\n' && left > 0) {
5199 cp++;
5200 left--;
5201 }
5202 if (*cp == '\n' && left > 0)
5203 cp++;
5204 *ptr = cp;
5205 return(0);
5206 }
5207
5208 static char *
pci_describe_device(device_t dev)5209 pci_describe_device(device_t dev)
5210 {
5211 int vendor, device;
5212 char *desc, *vp, *dp, *line;
5213
5214 desc = vp = dp = NULL;
5215
5216 /*
5217 * If we have no vendor data, we can't do anything.
5218 */
5219 if (pci_vendordata == NULL)
5220 goto out;
5221
5222 /*
5223 * Scan the vendor data looking for this device
5224 */
5225 line = pci_vendordata;
5226 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5227 goto out;
5228 for (;;) {
5229 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5230 goto out;
5231 if (vendor == pci_get_vendor(dev))
5232 break;
5233 }
5234 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5235 goto out;
5236 for (;;) {
5237 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5238 *dp = 0;
5239 break;
5240 }
5241 if (vendor != -1) {
5242 *dp = 0;
5243 break;
5244 }
5245 if (device == pci_get_device(dev))
5246 break;
5247 }
5248 if (dp[0] == '\0')
5249 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5250 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5251 NULL)
5252 sprintf(desc, "%s, %s", vp, dp);
5253 out:
5254 if (vp != NULL)
5255 free(vp, M_DEVBUF);
5256 if (dp != NULL)
5257 free(dp, M_DEVBUF);
5258 return(desc);
5259 }
5260
5261 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5262 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5263 {
5264 struct pci_devinfo *dinfo;
5265 pcicfgregs *cfg;
5266
5267 dinfo = device_get_ivars(child);
5268 cfg = &dinfo->cfg;
5269
5270 switch (which) {
5271 case PCI_IVAR_ETHADDR:
5272 /*
5273 * The generic accessor doesn't deal with failure, so
5274 * we set the return value, then return an error.
5275 */
5276 *((uint8_t **) result) = NULL;
5277 return (EINVAL);
5278 case PCI_IVAR_SUBVENDOR:
5279 *result = cfg->subvendor;
5280 break;
5281 case PCI_IVAR_SUBDEVICE:
5282 *result = cfg->subdevice;
5283 break;
5284 case PCI_IVAR_VENDOR:
5285 *result = cfg->vendor;
5286 break;
5287 case PCI_IVAR_DEVICE:
5288 *result = cfg->device;
5289 break;
5290 case PCI_IVAR_DEVID:
5291 *result = (cfg->device << 16) | cfg->vendor;
5292 break;
5293 case PCI_IVAR_CLASS:
5294 *result = cfg->baseclass;
5295 break;
5296 case PCI_IVAR_SUBCLASS:
5297 *result = cfg->subclass;
5298 break;
5299 case PCI_IVAR_PROGIF:
5300 *result = cfg->progif;
5301 break;
5302 case PCI_IVAR_REVID:
5303 *result = cfg->revid;
5304 break;
5305 case PCI_IVAR_INTPIN:
5306 *result = cfg->intpin;
5307 break;
5308 case PCI_IVAR_IRQ:
5309 *result = cfg->intline;
5310 break;
5311 case PCI_IVAR_DOMAIN:
5312 *result = cfg->domain;
5313 break;
5314 case PCI_IVAR_BUS:
5315 *result = cfg->bus;
5316 break;
5317 case PCI_IVAR_SLOT:
5318 *result = cfg->slot;
5319 break;
5320 case PCI_IVAR_FUNCTION:
5321 *result = cfg->func;
5322 break;
5323 case PCI_IVAR_CMDREG:
5324 *result = cfg->cmdreg;
5325 break;
5326 case PCI_IVAR_CACHELNSZ:
5327 *result = cfg->cachelnsz;
5328 break;
5329 case PCI_IVAR_MINGNT:
5330 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5331 *result = -1;
5332 return (EINVAL);
5333 }
5334 *result = cfg->mingnt;
5335 break;
5336 case PCI_IVAR_MAXLAT:
5337 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5338 *result = -1;
5339 return (EINVAL);
5340 }
5341 *result = cfg->maxlat;
5342 break;
5343 case PCI_IVAR_LATTIMER:
5344 *result = cfg->lattimer;
5345 break;
5346 default:
5347 return (ENOENT);
5348 }
5349 return (0);
5350 }
5351
5352 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5353 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5354 {
5355 struct pci_devinfo *dinfo;
5356
5357 dinfo = device_get_ivars(child);
5358
5359 switch (which) {
5360 case PCI_IVAR_INTPIN:
5361 dinfo->cfg.intpin = value;
5362 return (0);
5363 case PCI_IVAR_ETHADDR:
5364 case PCI_IVAR_SUBVENDOR:
5365 case PCI_IVAR_SUBDEVICE:
5366 case PCI_IVAR_VENDOR:
5367 case PCI_IVAR_DEVICE:
5368 case PCI_IVAR_DEVID:
5369 case PCI_IVAR_CLASS:
5370 case PCI_IVAR_SUBCLASS:
5371 case PCI_IVAR_PROGIF:
5372 case PCI_IVAR_REVID:
5373 case PCI_IVAR_IRQ:
5374 case PCI_IVAR_DOMAIN:
5375 case PCI_IVAR_BUS:
5376 case PCI_IVAR_SLOT:
5377 case PCI_IVAR_FUNCTION:
5378 return (EINVAL); /* disallow for now */
5379
5380 default:
5381 return (ENOENT);
5382 }
5383 }
5384
5385 #include "opt_ddb.h"
5386 #ifdef DDB
5387 #include <ddb/ddb.h>
5388 #include <sys/cons.h>
5389
5390 /*
5391 * List resources based on pci map registers, used for within ddb
5392 */
5393
DB_SHOW_COMMAND(pciregs,db_pci_dump)5394 DB_SHOW_COMMAND(pciregs, db_pci_dump)
5395 {
5396 struct pci_devinfo *dinfo;
5397 struct devlist *devlist_head;
5398 struct pci_conf *p;
5399 const char *name;
5400 int i, error, none_count;
5401
5402 none_count = 0;
5403 /* get the head of the device queue */
5404 devlist_head = &pci_devq;
5405
5406 /*
5407 * Go through the list of devices and print out devices
5408 */
5409 for (error = 0, i = 0,
5410 dinfo = STAILQ_FIRST(devlist_head);
5411 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5412 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5413 /* Populate pd_name and pd_unit */
5414 name = NULL;
5415 if (dinfo->cfg.dev)
5416 name = device_get_name(dinfo->cfg.dev);
5417
5418 p = &dinfo->conf;
5419 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5420 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5421 (name && *name) ? name : "none",
5422 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5423 none_count++,
5424 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5425 p->pc_sel.pc_func, (p->pc_class << 16) |
5426 (p->pc_subclass << 8) | p->pc_progif,
5427 (p->pc_subdevice << 16) | p->pc_subvendor,
5428 (p->pc_device << 16) | p->pc_vendor,
5429 p->pc_revid, p->pc_hdr);
5430 }
5431 }
5432 #endif /* DDB */
5433
5434 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5435 pci_reserve_map(device_t dev, device_t child, int type, int *rid,
5436 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5437 u_int flags)
5438 {
5439 struct pci_devinfo *dinfo = device_get_ivars(child);
5440 struct resource_list *rl = &dinfo->resources;
5441 struct resource *res;
5442 struct pci_map *pm;
5443 uint16_t cmd;
5444 pci_addr_t map, testval;
5445 int mapsize;
5446
5447 res = NULL;
5448
5449 /* If rid is managed by EA, ignore it */
5450 if (pci_ea_is_enabled(child, *rid))
5451 goto out;
5452
5453 pm = pci_find_bar(child, *rid);
5454 if (pm != NULL) {
5455 /* This is a BAR that we failed to allocate earlier. */
5456 mapsize = pm->pm_size;
5457 map = pm->pm_value;
5458 } else {
5459 /*
5460 * Weed out the bogons, and figure out how large the
5461 * BAR/map is. BARs that read back 0 here are bogus
5462 * and unimplemented. Note: atapci in legacy mode are
5463 * special and handled elsewhere in the code. If you
5464 * have a atapci device in legacy mode and it fails
5465 * here, that other code is broken.
5466 */
5467 pci_read_bar(child, *rid, &map, &testval, NULL);
5468
5469 /*
5470 * Determine the size of the BAR and ignore BARs with a size
5471 * of 0. Device ROM BARs use a different mask value.
5472 */
5473 if (PCIR_IS_BIOS(&dinfo->cfg, *rid))
5474 mapsize = pci_romsize(testval);
5475 else
5476 mapsize = pci_mapsize(testval);
5477 if (mapsize == 0)
5478 goto out;
5479 pm = pci_add_bar(child, *rid, map, mapsize);
5480 }
5481
5482 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) {
5483 if (type != SYS_RES_MEMORY) {
5484 if (bootverbose)
5485 device_printf(dev,
5486 "child %s requested type %d for rid %#x,"
5487 " but the BAR says it is an memio\n",
5488 device_get_nameunit(child), type, *rid);
5489 goto out;
5490 }
5491 } else {
5492 if (type != SYS_RES_IOPORT) {
5493 if (bootverbose)
5494 device_printf(dev,
5495 "child %s requested type %d for rid %#x,"
5496 " but the BAR says it is an ioport\n",
5497 device_get_nameunit(child), type, *rid);
5498 goto out;
5499 }
5500 }
5501
5502 /*
5503 * For real BARs, we need to override the size that
5504 * the driver requests, because that's what the BAR
5505 * actually uses and we would otherwise have a
5506 * situation where we might allocate the excess to
5507 * another driver, which won't work.
5508 */
5509 count = ((pci_addr_t)1 << mapsize) * num;
5510 if (RF_ALIGNMENT(flags) < mapsize)
5511 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5512 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5513 flags |= RF_PREFETCHABLE;
5514
5515 /*
5516 * Allocate enough resource, and then write back the
5517 * appropriate BAR for that resource.
5518 */
5519 resource_list_add(rl, type, *rid, start, end, count);
5520 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5521 count, flags & ~RF_ACTIVE);
5522 if (res == NULL) {
5523 resource_list_delete(rl, type, *rid);
5524 device_printf(child,
5525 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5526 count, *rid, type, start, end);
5527 goto out;
5528 }
5529 if (bootverbose)
5530 device_printf(child,
5531 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5532 count, *rid, type, rman_get_start(res));
5533
5534 /* Disable decoding via the CMD register before updating the BAR */
5535 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5536 pci_write_config(child, PCIR_COMMAND,
5537 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5538
5539 map = rman_get_start(res);
5540 pci_write_bar(child, pm, map);
5541
5542 /* Restore the original value of the CMD register */
5543 pci_write_config(child, PCIR_COMMAND, cmd, 2);
5544 out:
5545 return (res);
5546 }
5547
5548 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)5549 pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid,
5550 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
5551 u_int flags)
5552 {
5553 struct pci_devinfo *dinfo;
5554 struct resource_list *rl;
5555 struct resource_list_entry *rle;
5556 struct resource *res;
5557 pcicfgregs *cfg;
5558
5559 /*
5560 * Perform lazy resource allocation
5561 */
5562 dinfo = device_get_ivars(child);
5563 rl = &dinfo->resources;
5564 cfg = &dinfo->cfg;
5565 switch (type) {
5566 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
5567 case PCI_RES_BUS:
5568 return (pci_alloc_secbus(dev, child, rid, start, end, count,
5569 flags));
5570 #endif
5571 case SYS_RES_IRQ:
5572 /*
5573 * Can't alloc legacy interrupt once MSI messages have
5574 * been allocated.
5575 */
5576 if (*rid == 0 && (cfg->msi.msi_alloc > 0 ||
5577 cfg->msix.msix_alloc > 0))
5578 return (NULL);
5579
5580 /*
5581 * If the child device doesn't have an interrupt
5582 * routed and is deserving of an interrupt, try to
5583 * assign it one.
5584 */
5585 if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
5586 (cfg->intpin != 0))
5587 pci_assign_interrupt(dev, child, 0);
5588 break;
5589 case SYS_RES_IOPORT:
5590 case SYS_RES_MEMORY:
5591 #ifdef NEW_PCIB
5592 /*
5593 * PCI-PCI bridge I/O window resources are not BARs.
5594 * For those allocations just pass the request up the
5595 * tree.
5596 */
5597 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
5598 switch (*rid) {
5599 case PCIR_IOBASEL_1:
5600 case PCIR_MEMBASE_1:
5601 case PCIR_PMBASEL_1:
5602 /*
5603 * XXX: Should we bother creating a resource
5604 * list entry?
5605 */
5606 return (bus_generic_alloc_resource(dev, child,
5607 type, rid, start, end, count, flags));
5608 }
5609 }
5610 #endif
5611 /* Reserve resources for this BAR if needed. */
5612 rle = resource_list_find(rl, type, *rid);
5613 if (rle == NULL) {
5614 res = pci_reserve_map(dev, child, type, rid, start, end,
5615 count, num, flags);
5616 if (res == NULL)
5617 return (NULL);
5618 }
5619 }
5620 return (resource_list_alloc(rl, dev, child, type, rid,
5621 start, end, count, flags));
5622 }
5623
5624 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)5625 pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
5626 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
5627 {
5628 #ifdef PCI_IOV
5629 struct pci_devinfo *dinfo;
5630 #endif
5631
5632 if (device_get_parent(child) != dev)
5633 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
5634 type, rid, start, end, count, flags));
5635
5636 #ifdef PCI_IOV
5637 dinfo = device_get_ivars(child);
5638 if (dinfo->cfg.flags & PCICFG_VF) {
5639 switch (type) {
5640 /* VFs can't have I/O BARs. */
5641 case SYS_RES_IOPORT:
5642 return (NULL);
5643 case SYS_RES_MEMORY:
5644 return (pci_vf_alloc_mem_resource(dev, child, rid,
5645 start, end, count, flags));
5646 }
5647
5648 /* Fall through for other types of resource allocations. */
5649 }
5650 #endif
5651
5652 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
5653 count, 1, flags));
5654 }
5655
5656 int
pci_release_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5657 pci_release_resource(device_t dev, device_t child, int type, int rid,
5658 struct resource *r)
5659 {
5660 struct pci_devinfo *dinfo;
5661 struct resource_list *rl;
5662 pcicfgregs *cfg __unused;
5663
5664 if (device_get_parent(child) != dev)
5665 return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child,
5666 type, rid, r));
5667
5668 dinfo = device_get_ivars(child);
5669 cfg = &dinfo->cfg;
5670
5671 #ifdef PCI_IOV
5672 if (cfg->flags & PCICFG_VF) {
5673 switch (type) {
5674 /* VFs can't have I/O BARs. */
5675 case SYS_RES_IOPORT:
5676 return (EDOOFUS);
5677 case SYS_RES_MEMORY:
5678 return (pci_vf_release_mem_resource(dev, child, rid,
5679 r));
5680 }
5681
5682 /* Fall through for other types of resource allocations. */
5683 }
5684 #endif
5685
5686 #ifdef NEW_PCIB
5687 /*
5688 * PCI-PCI bridge I/O window resources are not BARs. For
5689 * those allocations just pass the request up the tree.
5690 */
5691 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
5692 (type == SYS_RES_IOPORT || type == SYS_RES_MEMORY)) {
5693 switch (rid) {
5694 case PCIR_IOBASEL_1:
5695 case PCIR_MEMBASE_1:
5696 case PCIR_PMBASEL_1:
5697 return (bus_generic_release_resource(dev, child, type,
5698 rid, r));
5699 }
5700 }
5701 #endif
5702
5703 rl = &dinfo->resources;
5704 return (resource_list_release(rl, dev, child, type, rid, r));
5705 }
5706
5707 int
pci_activate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5708 pci_activate_resource(device_t dev, device_t child, int type, int rid,
5709 struct resource *r)
5710 {
5711 struct pci_devinfo *dinfo;
5712 int error;
5713
5714 error = bus_generic_activate_resource(dev, child, type, rid, r);
5715 if (error)
5716 return (error);
5717
5718 /* Enable decoding in the command register when activating BARs. */
5719 if (device_get_parent(child) == dev) {
5720 /* Device ROMs need their decoding explicitly enabled. */
5721 dinfo = device_get_ivars(child);
5722 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5723 pci_write_bar(child, pci_find_bar(child, rid),
5724 rman_get_start(r) | PCIM_BIOS_ENABLE);
5725 switch (type) {
5726 case SYS_RES_IOPORT:
5727 case SYS_RES_MEMORY:
5728 error = PCI_ENABLE_IO(dev, child, type);
5729 break;
5730 }
5731 }
5732 return (error);
5733 }
5734
5735 int
pci_deactivate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5736 pci_deactivate_resource(device_t dev, device_t child, int type,
5737 int rid, struct resource *r)
5738 {
5739 struct pci_devinfo *dinfo;
5740 int error;
5741
5742 error = bus_generic_deactivate_resource(dev, child, type, rid, r);
5743 if (error)
5744 return (error);
5745
5746 /* Disable decoding for device ROMs. */
5747 if (device_get_parent(child) == dev) {
5748 dinfo = device_get_ivars(child);
5749 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5750 pci_write_bar(child, pci_find_bar(child, rid),
5751 rman_get_start(r));
5752 }
5753 return (0);
5754 }
5755
5756 void
pci_child_deleted(device_t dev,device_t child)5757 pci_child_deleted(device_t dev, device_t child)
5758 {
5759 struct resource_list_entry *rle;
5760 struct resource_list *rl;
5761 struct pci_devinfo *dinfo;
5762
5763 dinfo = device_get_ivars(child);
5764 rl = &dinfo->resources;
5765
5766 EVENTHANDLER_INVOKE(pci_delete_device, child);
5767
5768 /* Turn off access to resources we're about to free */
5769 if (bus_child_present(child) != 0) {
5770 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
5771 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
5772
5773 pci_disable_busmaster(child);
5774 }
5775
5776 /* Free all allocated resources */
5777 STAILQ_FOREACH(rle, rl, link) {
5778 if (rle->res) {
5779 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5780 resource_list_busy(rl, rle->type, rle->rid)) {
5781 pci_printf(&dinfo->cfg,
5782 "Resource still owned, oops. "
5783 "(type=%d, rid=%d, addr=%lx)\n",
5784 rle->type, rle->rid,
5785 rman_get_start(rle->res));
5786 bus_release_resource(child, rle->type, rle->rid,
5787 rle->res);
5788 }
5789 resource_list_unreserve(rl, dev, child, rle->type,
5790 rle->rid);
5791 }
5792 }
5793 resource_list_free(rl);
5794
5795 pci_freecfg(dinfo);
5796 }
5797
5798 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)5799 pci_delete_resource(device_t dev, device_t child, int type, int rid)
5800 {
5801 struct pci_devinfo *dinfo;
5802 struct resource_list *rl;
5803 struct resource_list_entry *rle;
5804
5805 if (device_get_parent(child) != dev)
5806 return;
5807
5808 dinfo = device_get_ivars(child);
5809 rl = &dinfo->resources;
5810 rle = resource_list_find(rl, type, rid);
5811 if (rle == NULL)
5812 return;
5813
5814 if (rle->res) {
5815 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5816 resource_list_busy(rl, type, rid)) {
5817 device_printf(dev, "delete_resource: "
5818 "Resource still owned by child, oops. "
5819 "(type=%d, rid=%d, addr=%jx)\n",
5820 type, rid, rman_get_start(rle->res));
5821 return;
5822 }
5823 resource_list_unreserve(rl, dev, child, type, rid);
5824 }
5825 resource_list_delete(rl, type, rid);
5826 }
5827
5828 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)5829 pci_get_resource_list (device_t dev, device_t child)
5830 {
5831 struct pci_devinfo *dinfo = device_get_ivars(child);
5832
5833 return (&dinfo->resources);
5834 }
5835
5836 #ifdef IOMMU
5837 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5838 pci_get_dma_tag(device_t bus, device_t dev)
5839 {
5840 bus_dma_tag_t tag;
5841 struct pci_softc *sc;
5842
5843 if (device_get_parent(dev) == bus) {
5844 /* try iommu and return if it works */
5845 tag = iommu_get_dma_tag(bus, dev);
5846 } else
5847 tag = NULL;
5848 if (tag == NULL) {
5849 sc = device_get_softc(bus);
5850 tag = sc->sc_dma_tag;
5851 }
5852 return (tag);
5853 }
5854 #else
5855 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5856 pci_get_dma_tag(device_t bus, device_t dev)
5857 {
5858 struct pci_softc *sc = device_get_softc(bus);
5859
5860 return (sc->sc_dma_tag);
5861 }
5862 #endif
5863
5864 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)5865 pci_read_config_method(device_t dev, device_t child, int reg, int width)
5866 {
5867 struct pci_devinfo *dinfo = device_get_ivars(child);
5868 pcicfgregs *cfg = &dinfo->cfg;
5869
5870 #ifdef PCI_IOV
5871 /*
5872 * SR-IOV VFs don't implement the VID or DID registers, so we have to
5873 * emulate them here.
5874 */
5875 if (cfg->flags & PCICFG_VF) {
5876 if (reg == PCIR_VENDOR) {
5877 switch (width) {
5878 case 4:
5879 return (cfg->device << 16 | cfg->vendor);
5880 case 2:
5881 return (cfg->vendor);
5882 case 1:
5883 return (cfg->vendor & 0xff);
5884 default:
5885 return (0xffffffff);
5886 }
5887 } else if (reg == PCIR_DEVICE) {
5888 switch (width) {
5889 /* Note that an unaligned 4-byte read is an error. */
5890 case 2:
5891 return (cfg->device);
5892 case 1:
5893 return (cfg->device & 0xff);
5894 default:
5895 return (0xffffffff);
5896 }
5897 }
5898 }
5899 #endif
5900
5901 return (PCIB_READ_CONFIG(device_get_parent(dev),
5902 cfg->bus, cfg->slot, cfg->func, reg, width));
5903 }
5904
5905 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)5906 pci_write_config_method(device_t dev, device_t child, int reg,
5907 uint32_t val, int width)
5908 {
5909 struct pci_devinfo *dinfo = device_get_ivars(child);
5910 pcicfgregs *cfg = &dinfo->cfg;
5911
5912 PCIB_WRITE_CONFIG(device_get_parent(dev),
5913 cfg->bus, cfg->slot, cfg->func, reg, val, width);
5914 }
5915
5916 int
pci_child_location_str_method(device_t dev,device_t child,char * buf,size_t buflen)5917 pci_child_location_str_method(device_t dev, device_t child, char *buf,
5918 size_t buflen)
5919 {
5920
5921 snprintf(buf, buflen, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
5922 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
5923 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
5924 return (0);
5925 }
5926
5927 int
pci_child_pnpinfo_str_method(device_t dev,device_t child,char * buf,size_t buflen)5928 pci_child_pnpinfo_str_method(device_t dev, device_t child, char *buf,
5929 size_t buflen)
5930 {
5931 struct pci_devinfo *dinfo;
5932 pcicfgregs *cfg;
5933
5934 dinfo = device_get_ivars(child);
5935 cfg = &dinfo->cfg;
5936 snprintf(buf, buflen, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
5937 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
5938 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
5939 cfg->progif);
5940 return (0);
5941 }
5942
5943 int
pci_assign_interrupt_method(device_t dev,device_t child)5944 pci_assign_interrupt_method(device_t dev, device_t child)
5945 {
5946 struct pci_devinfo *dinfo = device_get_ivars(child);
5947 pcicfgregs *cfg = &dinfo->cfg;
5948
5949 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
5950 cfg->intpin));
5951 }
5952
5953 static void
pci_lookup(void * arg,const char * name,device_t * dev)5954 pci_lookup(void *arg, const char *name, device_t *dev)
5955 {
5956 long val;
5957 char *end;
5958 int domain, bus, slot, func;
5959
5960 if (*dev != NULL)
5961 return;
5962
5963 /*
5964 * Accept pciconf-style selectors of either pciD:B:S:F or
5965 * pciB:S:F. In the latter case, the domain is assumed to
5966 * be zero.
5967 */
5968 if (strncmp(name, "pci", 3) != 0)
5969 return;
5970 val = strtol(name + 3, &end, 10);
5971 if (val < 0 || val > INT_MAX || *end != ':')
5972 return;
5973 domain = val;
5974 val = strtol(end + 1, &end, 10);
5975 if (val < 0 || val > INT_MAX || *end != ':')
5976 return;
5977 bus = val;
5978 val = strtol(end + 1, &end, 10);
5979 if (val < 0 || val > INT_MAX)
5980 return;
5981 slot = val;
5982 if (*end == ':') {
5983 val = strtol(end + 1, &end, 10);
5984 if (val < 0 || val > INT_MAX || *end != '\0')
5985 return;
5986 func = val;
5987 } else if (*end == '\0') {
5988 func = slot;
5989 slot = bus;
5990 bus = domain;
5991 domain = 0;
5992 } else
5993 return;
5994
5995 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
5996 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
5997 return;
5998
5999 *dev = pci_find_dbsf(domain, bus, slot, func);
6000 }
6001
6002 static int
pci_modevent(module_t mod,int what,void * arg)6003 pci_modevent(module_t mod, int what, void *arg)
6004 {
6005 static struct cdev *pci_cdev;
6006 static eventhandler_tag tag;
6007
6008 switch (what) {
6009 case MOD_LOAD:
6010 STAILQ_INIT(&pci_devq);
6011 pci_generation = 0;
6012 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6013 "pci");
6014 pci_load_vendor_data();
6015 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6016 1000);
6017 break;
6018
6019 case MOD_UNLOAD:
6020 if (tag != NULL)
6021 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6022 destroy_dev(pci_cdev);
6023 break;
6024 }
6025
6026 return (0);
6027 }
6028
6029 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6030 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6031 {
6032 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6033 struct pcicfg_pcie *cfg;
6034 int version, pos;
6035
6036 cfg = &dinfo->cfg.pcie;
6037 pos = cfg->pcie_location;
6038
6039 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6040
6041 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6042
6043 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6044 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6045 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6046 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6047
6048 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6049 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6050 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6051 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6052
6053 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6054 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6055 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6056
6057 if (version > 1) {
6058 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6059 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6060 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6061 }
6062 #undef WREG
6063 }
6064
6065 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6066 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6067 {
6068 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6069 dinfo->cfg.pcix.pcix_command, 2);
6070 }
6071
6072 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6073 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6074 {
6075
6076 /*
6077 * Restore the device to full power mode. We must do this
6078 * before we restore the registers because moving from D3 to
6079 * D0 will cause the chip's BARs and some other registers to
6080 * be reset to some unknown power on reset values. Cut down
6081 * the noise on boot by doing nothing if we are already in
6082 * state D0.
6083 */
6084 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6085 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6086 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6087 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6088 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6089 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6090 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6091 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6092 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6093 case PCIM_HDRTYPE_NORMAL:
6094 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6095 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6096 break;
6097 case PCIM_HDRTYPE_BRIDGE:
6098 pci_write_config(dev, PCIR_SECLAT_1,
6099 dinfo->cfg.bridge.br_seclat, 1);
6100 pci_write_config(dev, PCIR_SUBBUS_1,
6101 dinfo->cfg.bridge.br_subbus, 1);
6102 pci_write_config(dev, PCIR_SECBUS_1,
6103 dinfo->cfg.bridge.br_secbus, 1);
6104 pci_write_config(dev, PCIR_PRIBUS_1,
6105 dinfo->cfg.bridge.br_pribus, 1);
6106 pci_write_config(dev, PCIR_BRIDGECTL_1,
6107 dinfo->cfg.bridge.br_control, 2);
6108 break;
6109 case PCIM_HDRTYPE_CARDBUS:
6110 pci_write_config(dev, PCIR_SECLAT_2,
6111 dinfo->cfg.bridge.br_seclat, 1);
6112 pci_write_config(dev, PCIR_SUBBUS_2,
6113 dinfo->cfg.bridge.br_subbus, 1);
6114 pci_write_config(dev, PCIR_SECBUS_2,
6115 dinfo->cfg.bridge.br_secbus, 1);
6116 pci_write_config(dev, PCIR_PRIBUS_2,
6117 dinfo->cfg.bridge.br_pribus, 1);
6118 pci_write_config(dev, PCIR_BRIDGECTL_2,
6119 dinfo->cfg.bridge.br_control, 2);
6120 break;
6121 }
6122 pci_restore_bars(dev);
6123
6124 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6125 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6126
6127 /*
6128 * Restore extended capabilities for PCI-Express and PCI-X
6129 */
6130 if (dinfo->cfg.pcie.pcie_location != 0)
6131 pci_cfg_restore_pcie(dev, dinfo);
6132 if (dinfo->cfg.pcix.pcix_location != 0)
6133 pci_cfg_restore_pcix(dev, dinfo);
6134
6135 /* Restore MSI and MSI-X configurations if they are present. */
6136 if (dinfo->cfg.msi.msi_location != 0)
6137 pci_resume_msi(dev);
6138 if (dinfo->cfg.msix.msix_location != 0)
6139 pci_resume_msix(dev);
6140
6141 #ifdef PCI_IOV
6142 if (dinfo->cfg.iov != NULL)
6143 pci_iov_cfg_restore(dev, dinfo);
6144 #endif
6145 }
6146
6147 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6148 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6149 {
6150 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6151 struct pcicfg_pcie *cfg;
6152 int version, pos;
6153
6154 cfg = &dinfo->cfg.pcie;
6155 pos = cfg->pcie_location;
6156
6157 cfg->pcie_flags = RREG(PCIER_FLAGS);
6158
6159 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6160
6161 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6162
6163 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6164 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6165 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6166 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6167
6168 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6169 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6170 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6171 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6172
6173 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6174 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6175 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6176
6177 if (version > 1) {
6178 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6179 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6180 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6181 }
6182 #undef RREG
6183 }
6184
6185 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6186 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6187 {
6188 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6189 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6190 }
6191
6192 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6193 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6194 {
6195 uint32_t cls;
6196 int ps;
6197
6198 /*
6199 * Some drivers apparently write to these registers w/o updating our
6200 * cached copy. No harm happens if we update the copy, so do so here
6201 * so we can restore them. The COMMAND register is modified by the
6202 * bus w/o updating the cache. This should represent the normally
6203 * writable portion of the 'defined' part of type 0/1/2 headers.
6204 */
6205 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6206 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6207 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6208 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6209 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6210 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6211 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6212 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6213 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6214 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6215 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6216 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6217 case PCIM_HDRTYPE_NORMAL:
6218 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6219 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6220 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6221 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6222 break;
6223 case PCIM_HDRTYPE_BRIDGE:
6224 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6225 PCIR_SECLAT_1, 1);
6226 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6227 PCIR_SUBBUS_1, 1);
6228 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6229 PCIR_SECBUS_1, 1);
6230 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6231 PCIR_PRIBUS_1, 1);
6232 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6233 PCIR_BRIDGECTL_1, 2);
6234 break;
6235 case PCIM_HDRTYPE_CARDBUS:
6236 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6237 PCIR_SECLAT_2, 1);
6238 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6239 PCIR_SUBBUS_2, 1);
6240 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6241 PCIR_SECBUS_2, 1);
6242 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6243 PCIR_PRIBUS_2, 1);
6244 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6245 PCIR_BRIDGECTL_2, 2);
6246 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6247 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6248 break;
6249 }
6250
6251 if (dinfo->cfg.pcie.pcie_location != 0)
6252 pci_cfg_save_pcie(dev, dinfo);
6253
6254 if (dinfo->cfg.pcix.pcix_location != 0)
6255 pci_cfg_save_pcix(dev, dinfo);
6256
6257 #ifdef PCI_IOV
6258 if (dinfo->cfg.iov != NULL)
6259 pci_iov_cfg_save(dev, dinfo);
6260 #endif
6261
6262 /*
6263 * don't set the state for display devices, base peripherals and
6264 * memory devices since bad things happen when they are powered down.
6265 * We should (a) have drivers that can easily detach and (b) use
6266 * generic drivers for these devices so that some device actually
6267 * attaches. We need to make sure that when we implement (a) we don't
6268 * power the device down on a reattach.
6269 */
6270 cls = pci_get_class(dev);
6271 if (!setstate)
6272 return;
6273 switch (pci_do_power_nodriver)
6274 {
6275 case 0: /* NO powerdown at all */
6276 return;
6277 case 1: /* Conservative about what to power down */
6278 if (cls == PCIC_STORAGE)
6279 return;
6280 /*FALLTHROUGH*/
6281 case 2: /* Aggressive about what to power down */
6282 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6283 cls == PCIC_BASEPERIPH)
6284 return;
6285 /*FALLTHROUGH*/
6286 case 3: /* Power down everything */
6287 break;
6288 }
6289 /*
6290 * PCI spec says we can only go into D3 state from D0 state.
6291 * Transition from D[12] into D0 before going to D3 state.
6292 */
6293 ps = pci_get_powerstate(dev);
6294 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6295 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6296 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6297 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6298 }
6299
6300 /* Wrapper APIs suitable for device driver use. */
6301 void
pci_save_state(device_t dev)6302 pci_save_state(device_t dev)
6303 {
6304 struct pci_devinfo *dinfo;
6305
6306 dinfo = device_get_ivars(dev);
6307 pci_cfg_save(dev, dinfo, 0);
6308 }
6309
6310 void
pci_restore_state(device_t dev)6311 pci_restore_state(device_t dev)
6312 {
6313 struct pci_devinfo *dinfo;
6314
6315 dinfo = device_get_ivars(dev);
6316 pci_cfg_restore(dev, dinfo);
6317 }
6318
6319 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6320 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6321 uintptr_t *id)
6322 {
6323
6324 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6325 }
6326
6327 /* Find the upstream port of a given PCI device in a root complex. */
6328 device_t
pci_find_pcie_root_port(device_t dev)6329 pci_find_pcie_root_port(device_t dev)
6330 {
6331 struct pci_devinfo *dinfo;
6332 devclass_t pci_class;
6333 device_t pcib, bus;
6334
6335 pci_class = devclass_find("pci");
6336 KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class,
6337 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6338
6339 /*
6340 * Walk the bridge hierarchy until we find a PCI-e root
6341 * port or a non-PCI device.
6342 */
6343 for (;;) {
6344 bus = device_get_parent(dev);
6345 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6346 device_get_nameunit(dev)));
6347
6348 pcib = device_get_parent(bus);
6349 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6350 device_get_nameunit(bus)));
6351
6352 /*
6353 * pcib's parent must be a PCI bus for this to be a
6354 * PCI-PCI bridge.
6355 */
6356 if (device_get_devclass(device_get_parent(pcib)) != pci_class)
6357 return (NULL);
6358
6359 dinfo = device_get_ivars(pcib);
6360 if (dinfo->cfg.pcie.pcie_location != 0 &&
6361 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6362 return (pcib);
6363
6364 dev = pcib;
6365 }
6366 }
6367
6368 /*
6369 * Wait for pending transactions to complete on a PCI-express function.
6370 *
6371 * The maximum delay is specified in milliseconds in max_delay. Note
6372 * that this function may sleep.
6373 *
6374 * Returns true if the function is idle and false if the timeout is
6375 * exceeded. If dev is not a PCI-express function, this returns true.
6376 */
6377 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6378 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6379 {
6380 struct pci_devinfo *dinfo = device_get_ivars(dev);
6381 uint16_t sta;
6382 int cap;
6383
6384 cap = dinfo->cfg.pcie.pcie_location;
6385 if (cap == 0)
6386 return (true);
6387
6388 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6389 while (sta & PCIEM_STA_TRANSACTION_PND) {
6390 if (max_delay == 0)
6391 return (false);
6392
6393 /* Poll once every 100 milliseconds up to the timeout. */
6394 if (max_delay > 100) {
6395 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6396 max_delay -= 100;
6397 } else {
6398 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6399 C_HARDCLOCK);
6400 max_delay = 0;
6401 }
6402 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6403 }
6404
6405 return (true);
6406 }
6407
6408 /*
6409 * Determine the maximum Completion Timeout in microseconds.
6410 *
6411 * For non-PCI-express functions this returns 0.
6412 */
6413 int
pcie_get_max_completion_timeout(device_t dev)6414 pcie_get_max_completion_timeout(device_t dev)
6415 {
6416 struct pci_devinfo *dinfo = device_get_ivars(dev);
6417 int cap;
6418
6419 cap = dinfo->cfg.pcie.pcie_location;
6420 if (cap == 0)
6421 return (0);
6422
6423 /*
6424 * Functions using the 1.x spec use the default timeout range of
6425 * 50 microseconds to 50 milliseconds. Functions that do not
6426 * support programmable timeouts also use this range.
6427 */
6428 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
6429 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
6430 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
6431 return (50 * 1000);
6432
6433 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
6434 PCIEM_CTL2_COMP_TIMO_VAL) {
6435 case PCIEM_CTL2_COMP_TIMO_100US:
6436 return (100);
6437 case PCIEM_CTL2_COMP_TIMO_10MS:
6438 return (10 * 1000);
6439 case PCIEM_CTL2_COMP_TIMO_55MS:
6440 return (55 * 1000);
6441 case PCIEM_CTL2_COMP_TIMO_210MS:
6442 return (210 * 1000);
6443 case PCIEM_CTL2_COMP_TIMO_900MS:
6444 return (900 * 1000);
6445 case PCIEM_CTL2_COMP_TIMO_3500MS:
6446 return (3500 * 1000);
6447 case PCIEM_CTL2_COMP_TIMO_13S:
6448 return (13 * 1000 * 1000);
6449 case PCIEM_CTL2_COMP_TIMO_64S:
6450 return (64 * 1000 * 1000);
6451 default:
6452 return (50 * 1000);
6453 }
6454 }
6455
6456 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)6457 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
6458 {
6459 struct pci_devinfo *dinfo = device_get_ivars(dev);
6460 const char *s;
6461 int aer;
6462 uint32_t r, r1;
6463 uint16_t rs;
6464
6465 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
6466 s = "Correctable";
6467 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
6468 s = "Uncorrectable (Non-Fatal)";
6469 else
6470 s = "Uncorrectable (Fatal)";
6471 device_printf(dev, "%s PCIe error reported by APEI\n", s);
6472 if (aerp) {
6473 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
6474 r = le32dec(aerp + PCIR_AER_COR_STATUS);
6475 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
6476 } else {
6477 r = le32dec(aerp + PCIR_AER_UC_STATUS);
6478 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
6479 }
6480 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
6481 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
6482 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
6483 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
6484 printf(" severity 0x%08x first %d\n",
6485 r, rs & 0x1f);
6486 } else
6487 printf("\n");
6488 }
6489
6490 /* As kind of recovery just report and clear the error statuses. */
6491 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6492 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6493 if (r != 0) {
6494 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6495 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
6496 }
6497
6498 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6499 if (r != 0) {
6500 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6501 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
6502 }
6503 }
6504 if (dinfo->cfg.pcie.pcie_location != 0) {
6505 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6506 PCIER_DEVICE_STA, 2);
6507 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
6508 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6509 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6510 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6511 PCIER_DEVICE_STA, rs, 2);
6512 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
6513 }
6514 }
6515 }
6516
6517 /*
6518 * Perform a Function Level Reset (FLR) on a device.
6519 *
6520 * This function first waits for any pending transactions to complete
6521 * within the timeout specified by max_delay. If transactions are
6522 * still pending, the function will return false without attempting a
6523 * reset.
6524 *
6525 * If dev is not a PCI-express function or does not support FLR, this
6526 * function returns false.
6527 *
6528 * Note that no registers are saved or restored. The caller is
6529 * responsible for saving and restoring any registers including
6530 * PCI-standard registers via pci_save_state() and
6531 * pci_restore_state().
6532 */
6533 bool
pcie_flr(device_t dev,u_int max_delay,bool force)6534 pcie_flr(device_t dev, u_int max_delay, bool force)
6535 {
6536 struct pci_devinfo *dinfo = device_get_ivars(dev);
6537 uint16_t cmd, ctl;
6538 int compl_delay;
6539 int cap;
6540
6541 cap = dinfo->cfg.pcie.pcie_location;
6542 if (cap == 0)
6543 return (false);
6544
6545 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR))
6546 return (false);
6547
6548 /*
6549 * Disable busmastering to prevent generation of new
6550 * transactions while waiting for the device to go idle. If
6551 * the idle timeout fails, the command register is restored
6552 * which will re-enable busmastering.
6553 */
6554 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
6555 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
6556 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
6557 if (!force) {
6558 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
6559 return (false);
6560 }
6561 pci_printf(&dinfo->cfg,
6562 "Resetting with transactions pending after %d ms\n",
6563 max_delay);
6564
6565 /*
6566 * Extend the post-FLR delay to cover the maximum
6567 * Completion Timeout delay of anything in flight
6568 * during the FLR delay. Enforce a minimum delay of
6569 * at least 10ms.
6570 */
6571 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
6572 if (compl_delay < 10)
6573 compl_delay = 10;
6574 } else
6575 compl_delay = 0;
6576
6577 /* Initiate the reset. */
6578 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
6579 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
6580 PCIEM_CTL_INITIATE_FLR, 2);
6581
6582 /* Wait for 100ms. */
6583 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
6584
6585 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
6586 PCIEM_STA_TRANSACTION_PND)
6587 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
6588 return (true);
6589 }
6590
6591 /*
6592 * Attempt a power-management reset by cycling the device in/out of D3
6593 * state. PCI spec says we can only go into D3 state from D0 state.
6594 * Transition from D[12] into D0 before going to D3 state.
6595 */
6596 int
pci_power_reset(device_t dev)6597 pci_power_reset(device_t dev)
6598 {
6599 int ps;
6600
6601 ps = pci_get_powerstate(dev);
6602 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6603 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6604 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6605 pci_set_powerstate(dev, ps);
6606 return (0);
6607 }
6608
6609 /*
6610 * Try link drop and retrain of the downstream port of upstream
6611 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
6612 * cause Conventional Hot reset of the device in the slot.
6613 * Alternative, for PCIe, could be the secondary bus reset initiatied
6614 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
6615 */
6616 int
pcie_link_reset(device_t port,int pcie_location)6617 pcie_link_reset(device_t port, int pcie_location)
6618 {
6619 uint16_t v;
6620
6621 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
6622 v |= PCIEM_LINK_CTL_LINK_DIS;
6623 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6624 pause_sbt("pcier1", mstosbt(20), 0, 0);
6625 v &= ~PCIEM_LINK_CTL_LINK_DIS;
6626 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
6627 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6628 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
6629 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
6630 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
6631 }
6632
6633 static int
pci_reset_post(device_t dev,device_t child)6634 pci_reset_post(device_t dev, device_t child)
6635 {
6636
6637 if (dev == device_get_parent(child))
6638 pci_restore_state(child);
6639 return (0);
6640 }
6641
6642 static int
pci_reset_prepare(device_t dev,device_t child)6643 pci_reset_prepare(device_t dev, device_t child)
6644 {
6645
6646 if (dev == device_get_parent(child))
6647 pci_save_state(child);
6648 return (0);
6649 }
6650
6651 static int
pci_reset_child(device_t dev,device_t child,int flags)6652 pci_reset_child(device_t dev, device_t child, int flags)
6653 {
6654 int error;
6655
6656 if (dev == NULL || device_get_parent(child) != dev)
6657 return (0);
6658 if ((flags & DEVF_RESET_DETACH) != 0) {
6659 error = device_get_state(child) == DS_ATTACHED ?
6660 device_detach(child) : 0;
6661 } else {
6662 error = BUS_SUSPEND_CHILD(dev, child);
6663 }
6664 if (error == 0) {
6665 if (!pcie_flr(child, 1000, false)) {
6666 error = BUS_RESET_PREPARE(dev, child);
6667 if (error == 0)
6668 pci_power_reset(child);
6669 BUS_RESET_POST(dev, child);
6670 }
6671 if ((flags & DEVF_RESET_DETACH) != 0)
6672 device_probe_and_attach(child);
6673 else
6674 BUS_RESUME_CHILD(dev, child);
6675 }
6676 return (error);
6677 }
6678
6679 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)6680 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
6681 {
6682 bool match;
6683 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
6684
6685 vendor = pci_get_vendor(child);
6686 device = pci_get_device(child);
6687 subvendor = pci_get_subvendor(child);
6688 subdevice = pci_get_subdevice(child);
6689 class = pci_get_class(child);
6690 subclass = pci_get_subclass(child);
6691 revid = pci_get_revid(child);
6692 while (nelt-- > 0) {
6693 match = true;
6694 if (id->match_flag_vendor)
6695 match &= vendor == id->vendor;
6696 if (id->match_flag_device)
6697 match &= device == id->device;
6698 if (id->match_flag_subvendor)
6699 match &= subvendor == id->subvendor;
6700 if (id->match_flag_subdevice)
6701 match &= subdevice == id->subdevice;
6702 if (id->match_flag_class)
6703 match &= class == id->class_id;
6704 if (id->match_flag_subclass)
6705 match &= subclass == id->subclass;
6706 if (id->match_flag_revid)
6707 match &= revid == id->revid;
6708 if (match)
6709 return (id);
6710 id++;
6711 }
6712 return (NULL);
6713 }
6714
6715 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)6716 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
6717 {
6718 const char *dev_name;
6719 device_t dev;
6720
6721 dev = dinfo->cfg.dev;
6722 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
6723 dinfo->cfg.slot, dinfo->cfg.func);
6724 dev_name = device_get_name(dev);
6725 if (dev_name != NULL)
6726 printf(" (%s%d)", dev_name, device_get_unit(dev));
6727 }
6728
6729 void
pci_print_faulted_dev(void)6730 pci_print_faulted_dev(void)
6731 {
6732 struct pci_devinfo *dinfo;
6733 device_t dev;
6734 int aer, i;
6735 uint32_t r1, r2;
6736 uint16_t status;
6737
6738 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6739 dev = dinfo->cfg.dev;
6740 status = pci_read_config(dev, PCIR_STATUS, 2);
6741 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6742 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6743 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6744 if (status != 0) {
6745 pci_print_faulted_dev_name(dinfo);
6746 printf(" error 0x%04x\n", status);
6747 }
6748 if (dinfo->cfg.pcie.pcie_location != 0) {
6749 status = pci_read_config(dev,
6750 dinfo->cfg.pcie.pcie_location +
6751 PCIER_DEVICE_STA, 2);
6752 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
6753 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6754 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6755 pci_print_faulted_dev_name(dinfo);
6756 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
6757 pci_read_config(dev,
6758 dinfo->cfg.pcie.pcie_location +
6759 PCIER_DEVICE_CTL, 2),
6760 status);
6761 }
6762 }
6763 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6764 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6765 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6766 if (r1 != 0 || r2 != 0) {
6767 pci_print_faulted_dev_name(dinfo);
6768 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
6769 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
6770 r1, pci_read_config(dev, aer +
6771 PCIR_AER_UC_MASK, 4),
6772 pci_read_config(dev, aer +
6773 PCIR_AER_UC_SEVERITY, 4),
6774 r2, pci_read_config(dev, aer +
6775 PCIR_AER_COR_MASK, 4),
6776 pci_read_config(dev, aer +
6777 PCIR_AER_CAP_CONTROL, 4));
6778 for (i = 0; i < 4; i++) {
6779 r1 = pci_read_config(dev, aer +
6780 PCIR_AER_HEADER_LOG + i * 4, 4);
6781 printf(" HL%d: 0x%08x\n", i, r1);
6782 }
6783 }
6784 }
6785 }
6786 }
6787
6788 #ifdef DDB
DB_SHOW_COMMAND(pcierr,pci_print_faulted_dev_db)6789 DB_SHOW_COMMAND(pcierr, pci_print_faulted_dev_db)
6790 {
6791
6792 pci_print_faulted_dev();
6793 }
6794
6795 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)6796 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
6797 {
6798 device_t dev;
6799 int aer;
6800 uint32_t r;
6801
6802 dev = dinfo->cfg.dev;
6803 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6804 PCIER_DEVICE_STA, 2);
6805 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6806 PCIER_DEVICE_STA, r, 2);
6807
6808 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
6809 return;
6810 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6811 if (r != 0)
6812 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6813 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6814 if (r != 0)
6815 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6816 }
6817
DB_COMMAND(pci_clearerr,db_pci_clearerr)6818 DB_COMMAND(pci_clearerr, db_pci_clearerr)
6819 {
6820 struct pci_devinfo *dinfo;
6821 device_t dev;
6822 uint16_t status, status1;
6823
6824 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6825 dev = dinfo->cfg.dev;
6826 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
6827 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6828 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6829 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6830 if (status1 != 0) {
6831 status &= ~status1;
6832 pci_write_config(dev, PCIR_STATUS, status, 2);
6833 }
6834 if (dinfo->cfg.pcie.pcie_location != 0)
6835 db_clear_pcie_errors(dinfo);
6836 }
6837 }
6838 #endif
6839