1 /*-
2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/malloc.h>
33 #include <sys/kernel.h>
34 #include <sys/sysctl.h>
35 #include <sys/lock.h>
36 #include <sys/mutex.h>
37 #include <sys/bus.h>
38 #include <sys/fcntl.h>
39 #include <sys/file.h>
40 #include <sys/filio.h>
41 #include <sys/pciio.h>
42 #include <sys/rwlock.h>
43
44 #include <vm/vm.h>
45 #include <vm/pmap.h>
46
47 #include <machine/stdarg.h>
48
49 #include <dev/pci/pcivar.h>
50 #include <dev/pci/pci_private.h>
51 #include <dev/pci/pci_iov.h>
52
53 #include <linux/kobject.h>
54 #include <linux/device.h>
55 #include <linux/slab.h>
56 #include <linux/module.h>
57 #include <linux/cdev.h>
58 #include <linux/file.h>
59 #include <linux/sysfs.h>
60 #include <linux/mm.h>
61 #include <linux/io.h>
62 #include <linux/vmalloc.h>
63 #include <linux/pci.h>
64 #include <linux/compat.h>
65
66 static device_probe_t linux_pci_probe;
67 static device_attach_t linux_pci_attach;
68 static device_detach_t linux_pci_detach;
69 static device_suspend_t linux_pci_suspend;
70 static device_resume_t linux_pci_resume;
71 static device_shutdown_t linux_pci_shutdown;
72 static pci_iov_init_t linux_pci_iov_init;
73 static pci_iov_uninit_t linux_pci_iov_uninit;
74 static pci_iov_add_vf_t linux_pci_iov_add_vf;
75
76 static device_method_t pci_methods[] = {
77 DEVMETHOD(device_probe, linux_pci_probe),
78 DEVMETHOD(device_attach, linux_pci_attach),
79 DEVMETHOD(device_detach, linux_pci_detach),
80 DEVMETHOD(device_suspend, linux_pci_suspend),
81 DEVMETHOD(device_resume, linux_pci_resume),
82 DEVMETHOD(device_shutdown, linux_pci_shutdown),
83 DEVMETHOD(pci_iov_init, linux_pci_iov_init),
84 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit),
85 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf),
86 DEVMETHOD_END
87 };
88
89 static struct pci_driver *
linux_pci_find(device_t dev,const struct pci_device_id ** idp)90 linux_pci_find(device_t dev, const struct pci_device_id **idp)
91 {
92 const struct pci_device_id *id;
93 struct pci_driver *pdrv;
94 uint16_t vendor;
95 uint16_t device;
96 uint16_t subvendor;
97 uint16_t subdevice;
98
99 vendor = pci_get_vendor(dev);
100 device = pci_get_device(dev);
101 subvendor = pci_get_subvendor(dev);
102 subdevice = pci_get_subdevice(dev);
103
104 spin_lock(&pci_lock);
105 list_for_each_entry(pdrv, &pci_drivers, links) {
106 for (id = pdrv->id_table; id->vendor != 0; id++) {
107 if (vendor == id->vendor &&
108 (PCI_ANY_ID == id->device || device == id->device) &&
109 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) &&
110 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) {
111 *idp = id;
112 spin_unlock(&pci_lock);
113 return (pdrv);
114 }
115 }
116 }
117 spin_unlock(&pci_lock);
118 return (NULL);
119 }
120
121 static int
linux_pci_probe(device_t dev)122 linux_pci_probe(device_t dev)
123 {
124 const struct pci_device_id *id;
125 struct pci_driver *pdrv;
126
127 if ((pdrv = linux_pci_find(dev, &id)) == NULL)
128 return (ENXIO);
129 if (device_get_driver(dev) != &pdrv->bsddriver)
130 return (ENXIO);
131 device_set_desc(dev, pdrv->name);
132 return (0);
133 }
134
135 static int
linux_pci_attach(device_t dev)136 linux_pci_attach(device_t dev)
137 {
138 struct resource_list_entry *rle;
139 struct pci_bus *pbus;
140 struct pci_dev *pdev;
141 struct pci_devinfo *dinfo;
142 struct pci_driver *pdrv;
143 const struct pci_device_id *id;
144 device_t parent;
145 devclass_t devclass;
146 int error;
147
148 linux_set_current(curthread);
149
150 pdrv = linux_pci_find(dev, &id);
151 pdev = device_get_softc(dev);
152
153 parent = device_get_parent(dev);
154 devclass = device_get_devclass(parent);
155 if (pdrv->isdrm) {
156 dinfo = device_get_ivars(parent);
157 device_set_ivars(dev, dinfo);
158 } else {
159 dinfo = device_get_ivars(dev);
160 }
161
162 pdev->dev.parent = &linux_root_device;
163 pdev->dev.bsddev = dev;
164 INIT_LIST_HEAD(&pdev->dev.irqents);
165 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev));
166 pdev->device = dinfo->cfg.device;
167 pdev->vendor = dinfo->cfg.vendor;
168 pdev->subsystem_vendor = dinfo->cfg.subvendor;
169 pdev->subsystem_device = dinfo->cfg.subdevice;
170 pdev->class = pci_get_class(dev);
171 pdev->revision = pci_get_revid(dev);
172 pdev->dev.dma_mask = &pdev->dma_mask;
173 pdev->pdrv = pdrv;
174 kobject_init(&pdev->dev.kobj, &linux_dev_ktype);
175 kobject_set_name(&pdev->dev.kobj, device_get_nameunit(dev));
176 kobject_add(&pdev->dev.kobj, &linux_root_device.kobj,
177 kobject_name(&pdev->dev.kobj));
178 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0);
179 if (rle != NULL)
180 pdev->dev.irq = rle->start;
181 else
182 pdev->dev.irq = LINUX_IRQ_INVALID;
183 pdev->irq = pdev->dev.irq;
184
185 pbus = malloc(sizeof(*pbus), M_DEVBUF, M_WAITOK | M_ZERO);
186 pbus->self = pdev;
187 pbus->number = pci_get_bus(dev);
188 pdev->bus = pbus;
189
190 spin_lock(&pci_lock);
191 list_add(&pdev->links, &pci_devices);
192 spin_unlock(&pci_lock);
193
194 error = pdrv->probe(pdev, id);
195 if (error) {
196 free(pdev->bus, M_DEVBUF);
197 spin_lock(&pci_lock);
198 list_del(&pdev->links);
199 spin_unlock(&pci_lock);
200 put_device(&pdev->dev);
201 error = -error;
202 }
203 return (error);
204 }
205
206 static int
linux_pci_detach(device_t dev)207 linux_pci_detach(device_t dev)
208 {
209 struct pci_dev *pdev;
210
211 linux_set_current(curthread);
212 pdev = device_get_softc(dev);
213
214 pdev->pdrv->remove(pdev);
215 free(pdev->bus, M_DEVBUF);
216
217 spin_lock(&pci_lock);
218 list_del(&pdev->links);
219 spin_unlock(&pci_lock);
220 device_set_desc(dev, NULL);
221 put_device(&pdev->dev);
222
223 return (0);
224 }
225
226 static int
linux_pci_suspend(device_t dev)227 linux_pci_suspend(device_t dev)
228 {
229 const struct dev_pm_ops *pmops;
230 struct pm_message pm = { };
231 struct pci_dev *pdev;
232 int error;
233
234 error = 0;
235 linux_set_current(curthread);
236 pdev = device_get_softc(dev);
237 pmops = pdev->pdrv->driver.pm;
238
239 if (pdev->pdrv->suspend != NULL)
240 error = -pdev->pdrv->suspend(pdev, pm);
241 else if (pmops != NULL && pmops->suspend != NULL) {
242 error = -pmops->suspend(&pdev->dev);
243 if (error == 0 && pmops->suspend_late != NULL)
244 error = -pmops->suspend_late(&pdev->dev);
245 }
246 return (error);
247 }
248
249 static int
linux_pci_resume(device_t dev)250 linux_pci_resume(device_t dev)
251 {
252 const struct dev_pm_ops *pmops;
253 struct pci_dev *pdev;
254 int error;
255
256 error = 0;
257 linux_set_current(curthread);
258 pdev = device_get_softc(dev);
259 pmops = pdev->pdrv->driver.pm;
260
261 if (pdev->pdrv->resume != NULL)
262 error = -pdev->pdrv->resume(pdev);
263 else if (pmops != NULL && pmops->resume != NULL) {
264 if (pmops->resume_early != NULL)
265 error = -pmops->resume_early(&pdev->dev);
266 if (error == 0 && pmops->resume != NULL)
267 error = -pmops->resume(&pdev->dev);
268 }
269 return (error);
270 }
271
272 static int
linux_pci_shutdown(device_t dev)273 linux_pci_shutdown(device_t dev)
274 {
275 struct pci_dev *pdev;
276
277 linux_set_current(curthread);
278 pdev = device_get_softc(dev);
279 if (pdev->pdrv->shutdown != NULL)
280 pdev->pdrv->shutdown(pdev);
281 return (0);
282 }
283
284 static int
linux_pci_iov_init(device_t dev,uint16_t num_vfs,const nvlist_t * pf_config)285 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
286 {
287 struct pci_dev *pdev;
288 int error;
289
290 linux_set_current(curthread);
291 pdev = device_get_softc(dev);
292 if (pdev->pdrv->bsd_iov_init != NULL)
293 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config);
294 else
295 error = EINVAL;
296 return (error);
297 }
298
299 static void
linux_pci_iov_uninit(device_t dev)300 linux_pci_iov_uninit(device_t dev)
301 {
302 struct pci_dev *pdev;
303
304 linux_set_current(curthread);
305 pdev = device_get_softc(dev);
306 if (pdev->pdrv->bsd_iov_uninit != NULL)
307 pdev->pdrv->bsd_iov_uninit(dev);
308 }
309
310 static int
linux_pci_iov_add_vf(device_t dev,uint16_t vfnum,const nvlist_t * vf_config)311 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
312 {
313 struct pci_dev *pdev;
314 int error;
315
316 linux_set_current(curthread);
317 pdev = device_get_softc(dev);
318 if (pdev->pdrv->bsd_iov_add_vf != NULL)
319 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config);
320 else
321 error = EINVAL;
322 return (error);
323 }
324
325 static int
_linux_pci_register_driver(struct pci_driver * pdrv,devclass_t dc)326 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc)
327 {
328 int error;
329
330 linux_set_current(curthread);
331 spin_lock(&pci_lock);
332 list_add(&pdrv->links, &pci_drivers);
333 spin_unlock(&pci_lock);
334 pdrv->bsddriver.name = pdrv->name;
335 pdrv->bsddriver.methods = pci_methods;
336 pdrv->bsddriver.size = sizeof(struct pci_dev);
337
338 mtx_lock(&Giant);
339 error = devclass_add_driver(dc, &pdrv->bsddriver,
340 BUS_PASS_DEFAULT, &pdrv->bsdclass);
341 mtx_unlock(&Giant);
342 return (-error);
343 }
344
345 int
linux_pci_register_driver(struct pci_driver * pdrv)346 linux_pci_register_driver(struct pci_driver *pdrv)
347 {
348 devclass_t dc;
349
350 dc = devclass_find("pci");
351 if (dc == NULL)
352 return (-ENXIO);
353 pdrv->isdrm = false;
354 return (_linux_pci_register_driver(pdrv, dc));
355 }
356
357 int
linux_pci_register_drm_driver(struct pci_driver * pdrv)358 linux_pci_register_drm_driver(struct pci_driver *pdrv)
359 {
360 devclass_t dc;
361
362 dc = devclass_create("vgapci");
363 if (dc == NULL)
364 return (-ENXIO);
365 pdrv->isdrm = true;
366 pdrv->name = "drmn";
367 return (_linux_pci_register_driver(pdrv, dc));
368 }
369
370 void
linux_pci_unregister_driver(struct pci_driver * pdrv)371 linux_pci_unregister_driver(struct pci_driver *pdrv)
372 {
373 devclass_t bus;
374
375 bus = devclass_find("pci");
376
377 spin_lock(&pci_lock);
378 list_del(&pdrv->links);
379 spin_unlock(&pci_lock);
380 mtx_lock(&Giant);
381 if (bus != NULL)
382 devclass_delete_driver(bus, &pdrv->bsddriver);
383 mtx_unlock(&Giant);
384 }
385