1 /*-
2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD: stable/12/sys/compat/linuxkpi/common/src/linux_pci.c 369011 2021-01-15 07:28:38Z git2svn $");
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/bus.h>
33 #include <sys/malloc.h>
34 #include <sys/kernel.h>
35 #include <sys/sysctl.h>
36 #include <sys/lock.h>
37 #include <sys/mutex.h>
38 #include <sys/fcntl.h>
39 #include <sys/file.h>
40 #include <sys/filio.h>
41 #include <sys/pciio.h>
42 #include <sys/pctrie.h>
43 #include <sys/rwlock.h>
44
45 #include <vm/vm.h>
46 #include <vm/pmap.h>
47
48 #include <machine/stdarg.h>
49
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pci_private.h>
52 #include <dev/pci/pci_iov.h>
53
54 #include <linux/kobject.h>
55 #include <linux/device.h>
56 #include <linux/slab.h>
57 #include <linux/module.h>
58 #include <linux/cdev.h>
59 #include <linux/file.h>
60 #include <linux/sysfs.h>
61 #include <linux/mm.h>
62 #include <linux/io.h>
63 #include <linux/vmalloc.h>
64 #include <linux/pci.h>
65 #include <linux/compat.h>
66
67 #include "pcib_if.h"
68
69 static device_probe_t linux_pci_probe;
70 static device_attach_t linux_pci_attach;
71 static device_detach_t linux_pci_detach;
72 static device_suspend_t linux_pci_suspend;
73 static device_resume_t linux_pci_resume;
74 static device_shutdown_t linux_pci_shutdown;
75 static pci_iov_init_t linux_pci_iov_init;
76 static pci_iov_uninit_t linux_pci_iov_uninit;
77 static pci_iov_add_vf_t linux_pci_iov_add_vf;
78
79 static device_method_t pci_methods[] = {
80 DEVMETHOD(device_probe, linux_pci_probe),
81 DEVMETHOD(device_attach, linux_pci_attach),
82 DEVMETHOD(device_detach, linux_pci_detach),
83 DEVMETHOD(device_suspend, linux_pci_suspend),
84 DEVMETHOD(device_resume, linux_pci_resume),
85 DEVMETHOD(device_shutdown, linux_pci_shutdown),
86 DEVMETHOD(pci_iov_init, linux_pci_iov_init),
87 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit),
88 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf),
89 DEVMETHOD_END
90 };
91
92 struct linux_dma_priv {
93 uint64_t dma_mask;
94 struct mtx lock;
95 bus_dma_tag_t dmat;
96 struct pctrie ptree;
97 };
98 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock)
99 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock)
100
101 static int
linux_pdev_dma_init(struct pci_dev * pdev)102 linux_pdev_dma_init(struct pci_dev *pdev)
103 {
104 struct linux_dma_priv *priv;
105 int error;
106
107 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO);
108 pdev->dev.dma_priv = priv;
109
110 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF);
111
112 pctrie_init(&priv->ptree);
113
114 /* create a default DMA tag */
115 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64));
116 if (error) {
117 mtx_destroy(&priv->lock);
118 free(priv, M_DEVBUF);
119 pdev->dev.dma_priv = NULL;
120 }
121 return (error);
122 }
123
124 static int
linux_pdev_dma_uninit(struct pci_dev * pdev)125 linux_pdev_dma_uninit(struct pci_dev *pdev)
126 {
127 struct linux_dma_priv *priv;
128
129 priv = pdev->dev.dma_priv;
130 if (priv->dmat)
131 bus_dma_tag_destroy(priv->dmat);
132 mtx_destroy(&priv->lock);
133 free(priv, M_DEVBUF);
134 pdev->dev.dma_priv = NULL;
135 return (0);
136 }
137
138 int
linux_dma_tag_init(struct device * dev,u64 dma_mask)139 linux_dma_tag_init(struct device *dev, u64 dma_mask)
140 {
141 struct linux_dma_priv *priv;
142 int error;
143
144 priv = dev->dma_priv;
145
146 if (priv->dmat) {
147 if (priv->dma_mask == dma_mask)
148 return (0);
149
150 bus_dma_tag_destroy(priv->dmat);
151 }
152
153 priv->dma_mask = dma_mask;
154
155 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
156 1, 0, /* alignment, boundary */
157 dma_mask, /* lowaddr */
158 BUS_SPACE_MAXADDR, /* highaddr */
159 NULL, NULL, /* filtfunc, filtfuncarg */
160 BUS_SPACE_MAXSIZE, /* maxsize */
161 1, /* nsegments */
162 BUS_SPACE_MAXSIZE, /* maxsegsz */
163 0, /* flags */
164 NULL, NULL, /* lockfunc, lockfuncarg */
165 &priv->dmat);
166 return (-error);
167 }
168
169 static struct pci_driver *
linux_pci_find(device_t dev,const struct pci_device_id ** idp)170 linux_pci_find(device_t dev, const struct pci_device_id **idp)
171 {
172 const struct pci_device_id *id;
173 struct pci_driver *pdrv;
174 uint16_t vendor;
175 uint16_t device;
176 uint16_t subvendor;
177 uint16_t subdevice;
178
179 vendor = pci_get_vendor(dev);
180 device = pci_get_device(dev);
181 subvendor = pci_get_subvendor(dev);
182 subdevice = pci_get_subdevice(dev);
183
184 spin_lock(&pci_lock);
185 list_for_each_entry(pdrv, &pci_drivers, links) {
186 for (id = pdrv->id_table; id->vendor != 0; id++) {
187 if (vendor == id->vendor &&
188 (PCI_ANY_ID == id->device || device == id->device) &&
189 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) &&
190 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) {
191 *idp = id;
192 spin_unlock(&pci_lock);
193 return (pdrv);
194 }
195 }
196 }
197 spin_unlock(&pci_lock);
198 return (NULL);
199 }
200
201 static int
linux_pci_probe(device_t dev)202 linux_pci_probe(device_t dev)
203 {
204 const struct pci_device_id *id;
205 struct pci_driver *pdrv;
206
207 if ((pdrv = linux_pci_find(dev, &id)) == NULL)
208 return (ENXIO);
209 if (device_get_driver(dev) != &pdrv->bsddriver)
210 return (ENXIO);
211 device_set_desc(dev, pdrv->name);
212 return (0);
213 }
214
215 static int
linux_pci_attach(device_t dev)216 linux_pci_attach(device_t dev)
217 {
218 const struct pci_device_id *id;
219 struct pci_driver *pdrv;
220 struct pci_dev *pdev;
221
222 pdrv = linux_pci_find(dev, &id);
223 pdev = device_get_softc(dev);
224
225 MPASS(pdrv != NULL);
226 MPASS(pdev != NULL);
227
228 return (linux_pci_attach_device(dev, pdrv, id, pdev));
229 }
230
231 int
linux_pci_attach_device(device_t dev,struct pci_driver * pdrv,const struct pci_device_id * id,struct pci_dev * pdev)232 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv,
233 const struct pci_device_id *id, struct pci_dev *pdev)
234 {
235 struct resource_list_entry *rle;
236 struct pci_bus *pbus;
237 struct pci_devinfo *dinfo;
238 device_t parent;
239 uintptr_t rid;
240 int error;
241 bool isdrm;
242
243 linux_set_current(curthread);
244
245 parent = device_get_parent(dev);
246 isdrm = pdrv != NULL && pdrv->isdrm;
247
248 if (isdrm) {
249 dinfo = device_get_ivars(parent);
250 device_set_ivars(dev, dinfo);
251 } else {
252 dinfo = device_get_ivars(dev);
253 }
254
255 pdev->dev.parent = &linux_root_device;
256 pdev->dev.bsddev = dev;
257 INIT_LIST_HEAD(&pdev->dev.irqents);
258 if (isdrm)
259 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid);
260 else
261 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid);
262 pdev->devfn = rid;
263 pdev->device = dinfo->cfg.device;
264 pdev->vendor = dinfo->cfg.vendor;
265 pdev->subsystem_vendor = dinfo->cfg.subvendor;
266 pdev->subsystem_device = dinfo->cfg.subdevice;
267 pdev->class = pci_get_class(dev);
268 pdev->revision = pci_get_revid(dev);
269 pdev->pdrv = pdrv;
270 kobject_init(&pdev->dev.kobj, &linux_dev_ktype);
271 kobject_set_name(&pdev->dev.kobj, device_get_nameunit(dev));
272 kobject_add(&pdev->dev.kobj, &linux_root_device.kobj,
273 kobject_name(&pdev->dev.kobj));
274 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0);
275 if (rle != NULL)
276 pdev->dev.irq = rle->start;
277 else
278 pdev->dev.irq = LINUX_IRQ_INVALID;
279 pdev->irq = pdev->dev.irq;
280 error = linux_pdev_dma_init(pdev);
281 if (error)
282 goto out_dma_init;
283
284 TAILQ_INIT(&pdev->mmio);
285 pbus = malloc(sizeof(*pbus), M_DEVBUF, M_WAITOK | M_ZERO);
286 pbus->self = pdev;
287 pbus->number = pci_get_bus(dev);
288 pbus->domain = pci_get_domain(dev);
289 pdev->bus = pbus;
290
291 spin_lock(&pci_lock);
292 list_add(&pdev->links, &pci_devices);
293 spin_unlock(&pci_lock);
294
295 if (pdrv != NULL) {
296 error = pdrv->probe(pdev, id);
297 if (error)
298 goto out_probe;
299 }
300 return (0);
301
302 out_probe:
303 free(pdev->bus, M_DEVBUF);
304 linux_pdev_dma_uninit(pdev);
305 out_dma_init:
306 spin_lock(&pci_lock);
307 list_del(&pdev->links);
308 spin_unlock(&pci_lock);
309 put_device(&pdev->dev);
310 return (-error);
311 }
312
313 static int
linux_pci_detach(device_t dev)314 linux_pci_detach(device_t dev)
315 {
316 struct pci_dev *pdev;
317
318 pdev = device_get_softc(dev);
319
320 MPASS(pdev != NULL);
321
322 device_set_desc(dev, NULL);
323
324 return (linux_pci_detach_device(pdev));
325 }
326
327 int
linux_pci_detach_device(struct pci_dev * pdev)328 linux_pci_detach_device(struct pci_dev *pdev)
329 {
330
331 linux_set_current(curthread);
332
333 if (pdev->pdrv != NULL)
334 pdev->pdrv->remove(pdev);
335
336 free(pdev->bus, M_DEVBUF);
337 linux_pdev_dma_uninit(pdev);
338
339 spin_lock(&pci_lock);
340 list_del(&pdev->links);
341 spin_unlock(&pci_lock);
342 put_device(&pdev->dev);
343
344 return (0);
345 }
346
347 static int
linux_pci_suspend(device_t dev)348 linux_pci_suspend(device_t dev)
349 {
350 const struct dev_pm_ops *pmops;
351 struct pm_message pm = { };
352 struct pci_dev *pdev;
353 int error;
354
355 error = 0;
356 linux_set_current(curthread);
357 pdev = device_get_softc(dev);
358 pmops = pdev->pdrv->driver.pm;
359
360 if (pdev->pdrv->suspend != NULL)
361 error = -pdev->pdrv->suspend(pdev, pm);
362 else if (pmops != NULL && pmops->suspend != NULL) {
363 error = -pmops->suspend(&pdev->dev);
364 if (error == 0 && pmops->suspend_late != NULL)
365 error = -pmops->suspend_late(&pdev->dev);
366 }
367 return (error);
368 }
369
370 static int
linux_pci_resume(device_t dev)371 linux_pci_resume(device_t dev)
372 {
373 const struct dev_pm_ops *pmops;
374 struct pci_dev *pdev;
375 int error;
376
377 error = 0;
378 linux_set_current(curthread);
379 pdev = device_get_softc(dev);
380 pmops = pdev->pdrv->driver.pm;
381
382 if (pdev->pdrv->resume != NULL)
383 error = -pdev->pdrv->resume(pdev);
384 else if (pmops != NULL && pmops->resume != NULL) {
385 if (pmops->resume_early != NULL)
386 error = -pmops->resume_early(&pdev->dev);
387 if (error == 0 && pmops->resume != NULL)
388 error = -pmops->resume(&pdev->dev);
389 }
390 return (error);
391 }
392
393 static int
linux_pci_shutdown(device_t dev)394 linux_pci_shutdown(device_t dev)
395 {
396 struct pci_dev *pdev;
397
398 linux_set_current(curthread);
399 pdev = device_get_softc(dev);
400 if (pdev->pdrv->shutdown != NULL)
401 pdev->pdrv->shutdown(pdev);
402 return (0);
403 }
404
405 static int
linux_pci_iov_init(device_t dev,uint16_t num_vfs,const nvlist_t * pf_config)406 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
407 {
408 struct pci_dev *pdev;
409 int error;
410
411 linux_set_current(curthread);
412 pdev = device_get_softc(dev);
413 if (pdev->pdrv->bsd_iov_init != NULL)
414 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config);
415 else
416 error = EINVAL;
417 return (error);
418 }
419
420 static void
linux_pci_iov_uninit(device_t dev)421 linux_pci_iov_uninit(device_t dev)
422 {
423 struct pci_dev *pdev;
424
425 linux_set_current(curthread);
426 pdev = device_get_softc(dev);
427 if (pdev->pdrv->bsd_iov_uninit != NULL)
428 pdev->pdrv->bsd_iov_uninit(dev);
429 }
430
431 static int
linux_pci_iov_add_vf(device_t dev,uint16_t vfnum,const nvlist_t * vf_config)432 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
433 {
434 struct pci_dev *pdev;
435 int error;
436
437 linux_set_current(curthread);
438 pdev = device_get_softc(dev);
439 if (pdev->pdrv->bsd_iov_add_vf != NULL)
440 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config);
441 else
442 error = EINVAL;
443 return (error);
444 }
445
446 static int
_linux_pci_register_driver(struct pci_driver * pdrv,devclass_t dc)447 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc)
448 {
449 int error;
450
451 linux_set_current(curthread);
452 spin_lock(&pci_lock);
453 list_add(&pdrv->links, &pci_drivers);
454 spin_unlock(&pci_lock);
455 pdrv->bsddriver.name = pdrv->name;
456 pdrv->bsddriver.methods = pci_methods;
457 pdrv->bsddriver.size = sizeof(struct pci_dev);
458
459 mtx_lock(&Giant);
460 error = devclass_add_driver(dc, &pdrv->bsddriver,
461 BUS_PASS_DEFAULT, &pdrv->bsdclass);
462 mtx_unlock(&Giant);
463 return (-error);
464 }
465
466 int
linux_pci_register_driver(struct pci_driver * pdrv)467 linux_pci_register_driver(struct pci_driver *pdrv)
468 {
469 devclass_t dc;
470
471 dc = devclass_find("pci");
472 if (dc == NULL)
473 return (-ENXIO);
474 pdrv->isdrm = false;
475 return (_linux_pci_register_driver(pdrv, dc));
476 }
477
478 unsigned long
pci_resource_start(struct pci_dev * pdev,int bar)479 pci_resource_start(struct pci_dev *pdev, int bar)
480 {
481 struct resource_list_entry *rle;
482 rman_res_t newstart;
483 device_t dev;
484
485 if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
486 return (0);
487 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ?
488 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev;
489 if (BUS_TRANSLATE_RESOURCE(dev, rle->type, rle->start, &newstart)) {
490 device_printf(pdev->dev.bsddev, "translate of %#jx failed\n",
491 (uintmax_t)rle->start);
492 return (0);
493 }
494 return (newstart);
495 }
496
497 unsigned long
pci_resource_len(struct pci_dev * pdev,int bar)498 pci_resource_len(struct pci_dev *pdev, int bar)
499 {
500 struct resource_list_entry *rle;
501
502 if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
503 return (0);
504 return (rle->count);
505 }
506
507 int
linux_pci_register_drm_driver(struct pci_driver * pdrv)508 linux_pci_register_drm_driver(struct pci_driver *pdrv)
509 {
510 devclass_t dc;
511
512 dc = devclass_create("vgapci");
513 if (dc == NULL)
514 return (-ENXIO);
515 pdrv->isdrm = true;
516 pdrv->name = "drmn";
517 return (_linux_pci_register_driver(pdrv, dc));
518 }
519
520 void
linux_pci_unregister_driver(struct pci_driver * pdrv)521 linux_pci_unregister_driver(struct pci_driver *pdrv)
522 {
523 devclass_t bus;
524
525 bus = devclass_find("pci");
526
527 spin_lock(&pci_lock);
528 list_del(&pdrv->links);
529 spin_unlock(&pci_lock);
530 mtx_lock(&Giant);
531 if (bus != NULL)
532 devclass_delete_driver(bus, &pdrv->bsddriver);
533 mtx_unlock(&Giant);
534 }
535
536 void
linux_pci_unregister_drm_driver(struct pci_driver * pdrv)537 linux_pci_unregister_drm_driver(struct pci_driver *pdrv)
538 {
539 devclass_t bus;
540
541 bus = devclass_find("vgapci");
542
543 spin_lock(&pci_lock);
544 list_del(&pdrv->links);
545 spin_unlock(&pci_lock);
546 mtx_lock(&Giant);
547 if (bus != NULL)
548 devclass_delete_driver(bus, &pdrv->bsddriver);
549 mtx_unlock(&Giant);
550 }
551
552 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t));
553
554 struct linux_dma_obj {
555 void *vaddr;
556 uint64_t dma_addr;
557 bus_dmamap_t dmamap;
558 };
559
560 static uma_zone_t linux_dma_trie_zone;
561 static uma_zone_t linux_dma_obj_zone;
562
563 static void
linux_dma_init(void * arg)564 linux_dma_init(void *arg)
565 {
566
567 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie",
568 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL,
569 UMA_ALIGN_PTR, 0);
570 linux_dma_obj_zone = uma_zcreate("linux_dma_object",
571 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL,
572 UMA_ALIGN_PTR, 0);
573
574 }
575 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL);
576
577 static void
linux_dma_uninit(void * arg)578 linux_dma_uninit(void *arg)
579 {
580
581 uma_zdestroy(linux_dma_obj_zone);
582 uma_zdestroy(linux_dma_trie_zone);
583 }
584 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL);
585
586 static void *
linux_dma_trie_alloc(struct pctrie * ptree)587 linux_dma_trie_alloc(struct pctrie *ptree)
588 {
589
590 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT));
591 }
592
593 static void
linux_dma_trie_free(struct pctrie * ptree,void * node)594 linux_dma_trie_free(struct pctrie *ptree, void *node)
595 {
596
597 uma_zfree(linux_dma_trie_zone, node);
598 }
599
600
601 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc,
602 linux_dma_trie_free);
603
604 void *
linux_dma_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flag)605 linux_dma_alloc_coherent(struct device *dev, size_t size,
606 dma_addr_t *dma_handle, gfp_t flag)
607 {
608 struct linux_dma_priv *priv;
609 vm_paddr_t high;
610 size_t align;
611 void *mem;
612
613 if (dev == NULL || dev->dma_priv == NULL) {
614 *dma_handle = 0;
615 return (NULL);
616 }
617 priv = dev->dma_priv;
618 if (priv->dma_mask)
619 high = priv->dma_mask;
620 else if (flag & GFP_DMA32)
621 high = BUS_SPACE_MAXADDR_32BIT;
622 else
623 high = BUS_SPACE_MAXADDR;
624 align = PAGE_SIZE << get_order(size);
625 mem = (void *)kmem_alloc_contig(size, flag, 0, high, align, 0,
626 VM_MEMATTR_DEFAULT);
627 if (mem != NULL) {
628 *dma_handle = linux_dma_map_phys(dev, vtophys(mem), size);
629 if (*dma_handle == 0) {
630 kmem_free((vm_offset_t)mem, size);
631 mem = NULL;
632 }
633 } else {
634 *dma_handle = 0;
635 }
636 return (mem);
637 }
638
639 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
640 dma_addr_t
linux_dma_map_phys(struct device * dev,vm_paddr_t phys,size_t len)641 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len)
642 {
643 struct linux_dma_priv *priv;
644 struct linux_dma_obj *obj;
645 int error, nseg;
646 bus_dma_segment_t seg;
647
648 priv = dev->dma_priv;
649
650 /*
651 * If the resultant mapping will be entirely 1:1 with the
652 * physical address, short-circuit the remainder of the
653 * bus_dma API. This avoids tracking collisions in the pctrie
654 * with the additional benefit of reducing overhead.
655 */
656 if (bus_dma_id_mapped(priv->dmat, phys, len))
657 return (phys);
658
659 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT);
660 if (obj == NULL) {
661 return (0);
662 }
663
664 DMA_PRIV_LOCK(priv);
665 if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) {
666 DMA_PRIV_UNLOCK(priv);
667 uma_zfree(linux_dma_obj_zone, obj);
668 return (0);
669 }
670
671 nseg = -1;
672 if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len,
673 BUS_DMA_NOWAIT, &seg, &nseg) != 0) {
674 bus_dmamap_destroy(priv->dmat, obj->dmamap);
675 DMA_PRIV_UNLOCK(priv);
676 uma_zfree(linux_dma_obj_zone, obj);
677 return (0);
678 }
679
680 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
681 obj->dma_addr = seg.ds_addr;
682
683 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);
684 if (error != 0) {
685 bus_dmamap_unload(priv->dmat, obj->dmamap);
686 bus_dmamap_destroy(priv->dmat, obj->dmamap);
687 DMA_PRIV_UNLOCK(priv);
688 uma_zfree(linux_dma_obj_zone, obj);
689 return (0);
690 }
691 DMA_PRIV_UNLOCK(priv);
692 return (obj->dma_addr);
693 }
694 #else
695 dma_addr_t
linux_dma_map_phys(struct device * dev,vm_paddr_t phys,size_t len)696 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len)
697 {
698 return (phys);
699 }
700 #endif
701
702 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
703 void
linux_dma_unmap(struct device * dev,dma_addr_t dma_addr,size_t len)704 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len)
705 {
706 struct linux_dma_priv *priv;
707 struct linux_dma_obj *obj;
708
709 priv = dev->dma_priv;
710
711 if (pctrie_is_empty(&priv->ptree))
712 return;
713
714 DMA_PRIV_LOCK(priv);
715 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
716 if (obj == NULL) {
717 DMA_PRIV_UNLOCK(priv);
718 return;
719 }
720 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr);
721 bus_dmamap_unload(priv->dmat, obj->dmamap);
722 bus_dmamap_destroy(priv->dmat, obj->dmamap);
723 DMA_PRIV_UNLOCK(priv);
724
725 uma_zfree(linux_dma_obj_zone, obj);
726 }
727 #else
728 void
linux_dma_unmap(struct device * dev,dma_addr_t dma_addr,size_t len)729 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len)
730 {
731 }
732 #endif
733
734 int
linux_dma_map_sg_attrs(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,struct dma_attrs * attrs)735 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
736 enum dma_data_direction dir, struct dma_attrs *attrs)
737 {
738 struct linux_dma_priv *priv;
739 struct scatterlist *sg;
740 int i, nseg;
741 bus_dma_segment_t seg;
742
743 priv = dev->dma_priv;
744
745 DMA_PRIV_LOCK(priv);
746
747 /* create common DMA map in the first S/G entry */
748 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) {
749 DMA_PRIV_UNLOCK(priv);
750 return (0);
751 }
752
753 /* load all S/G list entries */
754 for_each_sg(sgl, sg, nents, i) {
755 nseg = -1;
756 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map,
757 sg_phys(sg), sg->length, BUS_DMA_NOWAIT,
758 &seg, &nseg) != 0) {
759 bus_dmamap_unload(priv->dmat, sgl->dma_map);
760 bus_dmamap_destroy(priv->dmat, sgl->dma_map);
761 DMA_PRIV_UNLOCK(priv);
762 return (0);
763 }
764 KASSERT(nseg == 0,
765 ("More than one segment (nseg=%d)", nseg + 1));
766
767 sg_dma_address(sg) = seg.ds_addr;
768 }
769 DMA_PRIV_UNLOCK(priv);
770
771 return (nents);
772 }
773
774 void
linux_dma_unmap_sg_attrs(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,struct dma_attrs * attrs)775 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
776 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
777 {
778 struct linux_dma_priv *priv;
779
780 priv = dev->dma_priv;
781
782 DMA_PRIV_LOCK(priv);
783 bus_dmamap_unload(priv->dmat, sgl->dma_map);
784 bus_dmamap_destroy(priv->dmat, sgl->dma_map);
785 DMA_PRIV_UNLOCK(priv);
786 }
787
788 struct dma_pool {
789 struct device *pool_device;
790 uma_zone_t pool_zone;
791 struct mtx pool_lock;
792 bus_dma_tag_t pool_dmat;
793 size_t pool_entry_size;
794 struct pctrie pool_ptree;
795 };
796
797 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock)
798 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock)
799
800 static inline int
dma_pool_obj_ctor(void * mem,int size,void * arg,int flags)801 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags)
802 {
803 struct linux_dma_obj *obj = mem;
804 struct dma_pool *pool = arg;
805 int error, nseg;
806 bus_dma_segment_t seg;
807
808 nseg = -1;
809 DMA_POOL_LOCK(pool);
810 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap,
811 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT,
812 &seg, &nseg);
813 DMA_POOL_UNLOCK(pool);
814 if (error != 0) {
815 return (error);
816 }
817 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
818 obj->dma_addr = seg.ds_addr;
819
820 return (0);
821 }
822
823 static void
dma_pool_obj_dtor(void * mem,int size,void * arg)824 dma_pool_obj_dtor(void *mem, int size, void *arg)
825 {
826 struct linux_dma_obj *obj = mem;
827 struct dma_pool *pool = arg;
828
829 DMA_POOL_LOCK(pool);
830 bus_dmamap_unload(pool->pool_dmat, obj->dmamap);
831 DMA_POOL_UNLOCK(pool);
832 }
833
834 static int
dma_pool_obj_import(void * arg,void ** store,int count,int domain __unused,int flags)835 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused,
836 int flags)
837 {
838 struct dma_pool *pool = arg;
839 struct linux_dma_priv *priv;
840 struct linux_dma_obj *obj;
841 int error, i;
842
843 priv = pool->pool_device->dma_priv;
844 for (i = 0; i < count; i++) {
845 obj = uma_zalloc(linux_dma_obj_zone, flags);
846 if (obj == NULL)
847 break;
848
849 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr,
850 BUS_DMA_NOWAIT, &obj->dmamap);
851 if (error!= 0) {
852 uma_zfree(linux_dma_obj_zone, obj);
853 break;
854 }
855
856 store[i] = obj;
857 }
858
859 return (i);
860 }
861
862 static void
dma_pool_obj_release(void * arg,void ** store,int count)863 dma_pool_obj_release(void *arg, void **store, int count)
864 {
865 struct dma_pool *pool = arg;
866 struct linux_dma_priv *priv;
867 struct linux_dma_obj *obj;
868 int i;
869
870 priv = pool->pool_device->dma_priv;
871 for (i = 0; i < count; i++) {
872 obj = store[i];
873 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap);
874 uma_zfree(linux_dma_obj_zone, obj);
875 }
876 }
877
878 struct dma_pool *
linux_dma_pool_create(char * name,struct device * dev,size_t size,size_t align,size_t boundary)879 linux_dma_pool_create(char *name, struct device *dev, size_t size,
880 size_t align, size_t boundary)
881 {
882 struct linux_dma_priv *priv;
883 struct dma_pool *pool;
884
885 priv = dev->dma_priv;
886
887 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
888 pool->pool_device = dev;
889 pool->pool_entry_size = size;
890
891 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
892 align, boundary, /* alignment, boundary */
893 priv->dma_mask, /* lowaddr */
894 BUS_SPACE_MAXADDR, /* highaddr */
895 NULL, NULL, /* filtfunc, filtfuncarg */
896 size, /* maxsize */
897 1, /* nsegments */
898 size, /* maxsegsz */
899 0, /* flags */
900 NULL, NULL, /* lockfunc, lockfuncarg */
901 &pool->pool_dmat)) {
902 kfree(pool);
903 return (NULL);
904 }
905
906 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor,
907 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import,
908 dma_pool_obj_release, pool, 0);
909
910 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF);
911 pctrie_init(&pool->pool_ptree);
912
913 return (pool);
914 }
915
916 void
linux_dma_pool_destroy(struct dma_pool * pool)917 linux_dma_pool_destroy(struct dma_pool *pool)
918 {
919
920 uma_zdestroy(pool->pool_zone);
921 bus_dma_tag_destroy(pool->pool_dmat);
922 mtx_destroy(&pool->pool_lock);
923 kfree(pool);
924 }
925
926 void *
linux_dma_pool_alloc(struct dma_pool * pool,gfp_t mem_flags,dma_addr_t * handle)927 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
928 dma_addr_t *handle)
929 {
930 struct linux_dma_obj *obj;
931
932 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags);
933 if (obj == NULL)
934 return (NULL);
935
936 DMA_POOL_LOCK(pool);
937 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) {
938 DMA_POOL_UNLOCK(pool);
939 uma_zfree_arg(pool->pool_zone, obj, pool);
940 return (NULL);
941 }
942 DMA_POOL_UNLOCK(pool);
943
944 *handle = obj->dma_addr;
945 return (obj->vaddr);
946 }
947
948 void
linux_dma_pool_free(struct dma_pool * pool,void * vaddr,dma_addr_t dma_addr)949 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr)
950 {
951 struct linux_dma_obj *obj;
952
953 DMA_POOL_LOCK(pool);
954 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr);
955 if (obj == NULL) {
956 DMA_POOL_UNLOCK(pool);
957 return;
958 }
959 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr);
960 DMA_POOL_UNLOCK(pool);
961
962 uma_zfree_arg(pool->pool_zone, obj, pool);
963 }
964