1 /*
2 * Copyright © 2012 Red Hat
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Dave Airlie <airlied@redhat.com>
25 * Rob Clark <rob.clark@linaro.org>
26 *
27 */
28
29 #include <linux/export.h>
30 #include <linux/dma-buf.h>
31 #include <linux/rbtree.h>
32 #include <drm/drm_prime.h>
33 #include <drm/drm_gem.h>
34 #include <drm/drmP.h>
35
36 #include "drm_internal.h"
37
38 /*
39 * DMA-BUF/GEM Object references and lifetime overview:
40 *
41 * On the export the dma_buf holds a reference to the exporting GEM
42 * object. It takes this reference in handle_to_fd_ioctl, when it
43 * first calls .prime_export and stores the exporting GEM object in
44 * the dma_buf priv. This reference needs to be released when the
45 * final reference to the &dma_buf itself is dropped and its
46 * &dma_buf_ops.release function is called. For GEM-based drivers,
47 * the dma_buf should be exported using drm_gem_dmabuf_export() and
48 * then released by drm_gem_dmabuf_release().
49 *
50 * On the import the importing GEM object holds a reference to the
51 * dma_buf (which in turn holds a ref to the exporting GEM object).
52 * It takes that reference in the fd_to_handle ioctl.
53 * It calls dma_buf_get, creates an attachment to it and stores the
54 * attachment in the GEM object. When this attachment is destroyed
55 * when the imported object is destroyed, we remove the attachment
56 * and drop the reference to the dma_buf.
57 *
58 * When all the references to the &dma_buf are dropped, i.e. when
59 * userspace has closed both handles to the imported GEM object (through the
60 * FD_TO_HANDLE IOCTL) and closed the file descriptor of the exported
61 * (through the HANDLE_TO_FD IOCTL) dma_buf, and all kernel-internal references
62 * are also gone, then the dma_buf gets destroyed. This can also happen as a
63 * part of the clean up procedure in the drm_release() function if userspace
64 * fails to properly clean up. Note that both the kernel and userspace (by
65 * keeeping the PRIME file descriptors open) can hold references onto a
66 * &dma_buf.
67 *
68 * Thus the chain of references always flows in one direction
69 * (avoiding loops): importing_gem -> dmabuf -> exporting_gem
70 *
71 * Self-importing: if userspace is using PRIME as a replacement for flink
72 * then it will get a fd->handle request for a GEM object that it created.
73 * Drivers should detect this situation and return back the gem object
74 * from the dma-buf private. Prime will do this automatically for drivers that
75 * use the drm_gem_prime_{import,export} helpers.
76 */
77
78 struct drm_prime_member {
79 struct dma_buf *dma_buf;
80 uint32_t handle;
81
82 struct rb_node dmabuf_rb;
83 struct rb_node handle_rb;
84 };
85
86 struct drm_prime_attachment {
87 struct sg_table *sgt;
88 enum dma_data_direction dir;
89 };
90
drm_prime_add_buf_handle(struct drm_prime_file_private * prime_fpriv,struct dma_buf * dma_buf,uint32_t handle)91 static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv,
92 struct dma_buf *dma_buf, uint32_t handle)
93 {
94 struct drm_prime_member *member;
95 struct rb_node **p, *rb;
96
97 member = kmalloc(sizeof(*member), M_DRM, GFP_KERNEL);
98 if (!member)
99 return -ENOMEM;
100
101 get_dma_buf(dma_buf);
102 member->dma_buf = dma_buf;
103 member->handle = handle;
104
105 rb = NULL;
106 p = &prime_fpriv->dmabufs.rb_node;
107 while (*p) {
108 struct drm_prime_member *pos;
109
110 rb = *p;
111 pos = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
112 if (dma_buf > pos->dma_buf)
113 p = &rb->rb_right;
114 else
115 p = &rb->rb_left;
116 }
117 rb_link_node(&member->dmabuf_rb, rb, p);
118 rb_insert_color(&member->dmabuf_rb, &prime_fpriv->dmabufs);
119
120 rb = NULL;
121 p = &prime_fpriv->handles.rb_node;
122 while (*p) {
123 struct drm_prime_member *pos;
124
125 rb = *p;
126 pos = rb_entry(rb, struct drm_prime_member, handle_rb);
127 if (handle > pos->handle)
128 p = &rb->rb_right;
129 else
130 p = &rb->rb_left;
131 }
132 rb_link_node(&member->handle_rb, rb, p);
133 rb_insert_color(&member->handle_rb, &prime_fpriv->handles);
134
135 return 0;
136 }
137
drm_prime_lookup_buf_by_handle(struct drm_prime_file_private * prime_fpriv,uint32_t handle)138 static struct dma_buf *drm_prime_lookup_buf_by_handle(struct drm_prime_file_private *prime_fpriv,
139 uint32_t handle)
140 {
141 struct rb_node *rb;
142
143 rb = prime_fpriv->handles.rb_node;
144 while (rb) {
145 struct drm_prime_member *member;
146
147 member = rb_entry(rb, struct drm_prime_member, handle_rb);
148 if (member->handle == handle)
149 return member->dma_buf;
150 else if (member->handle < handle)
151 rb = rb->rb_right;
152 else
153 rb = rb->rb_left;
154 }
155
156 return NULL;
157 }
158
drm_prime_lookup_buf_handle(struct drm_prime_file_private * prime_fpriv,struct dma_buf * dma_buf,uint32_t * handle)159 static int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpriv,
160 struct dma_buf *dma_buf,
161 uint32_t *handle)
162 {
163 struct rb_node *rb;
164
165 rb = prime_fpriv->dmabufs.rb_node;
166 while (rb) {
167 struct drm_prime_member *member;
168
169 member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
170 if (member->dma_buf == dma_buf) {
171 *handle = member->handle;
172 return 0;
173 } else if (member->dma_buf < dma_buf) {
174 rb = rb->rb_right;
175 } else {
176 rb = rb->rb_left;
177 }
178 }
179
180 return -ENOENT;
181 }
182
drm_gem_map_attach(struct dma_buf * dma_buf,struct device * target_dev,struct dma_buf_attachment * attach)183 static int drm_gem_map_attach(struct dma_buf *dma_buf,
184 struct device *target_dev,
185 struct dma_buf_attachment *attach)
186 {
187 struct drm_prime_attachment *prime_attach;
188 struct drm_gem_object *obj = dma_buf->priv;
189 struct drm_device *dev = obj->dev;
190
191 prime_attach = kzalloc(sizeof(*prime_attach), GFP_KERNEL);
192 if (!prime_attach)
193 return -ENOMEM;
194
195 prime_attach->dir = DMA_NONE;
196 attach->priv = prime_attach;
197
198 if (!dev->driver->gem_prime_pin)
199 return 0;
200
201 return dev->driver->gem_prime_pin(obj);
202 }
203
drm_gem_map_detach(struct dma_buf * dma_buf,struct dma_buf_attachment * attach)204 static void drm_gem_map_detach(struct dma_buf *dma_buf,
205 struct dma_buf_attachment *attach)
206 {
207 struct drm_prime_attachment *prime_attach = attach->priv;
208 struct drm_gem_object *obj = dma_buf->priv;
209 struct drm_device *dev = obj->dev;
210 struct sg_table *sgt;
211
212 if (dev->driver->gem_prime_unpin)
213 dev->driver->gem_prime_unpin(obj);
214
215 if (!prime_attach)
216 return;
217
218 sgt = prime_attach->sgt;
219 if (sgt) {
220 if (prime_attach->dir != DMA_NONE)
221 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
222 prime_attach->dir);
223 sg_free_table(sgt);
224 }
225
226 kfree(sgt);
227 kfree(prime_attach);
228 attach->priv = NULL;
229 }
230
drm_prime_remove_buf_handle_locked(struct drm_prime_file_private * prime_fpriv,struct dma_buf * dma_buf)231 void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv,
232 struct dma_buf *dma_buf)
233 {
234 struct rb_node *rb;
235
236 rb = prime_fpriv->dmabufs.rb_node;
237 while (rb) {
238 struct drm_prime_member *member;
239
240 member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
241 if (member->dma_buf == dma_buf) {
242 rb_erase(&member->handle_rb, &prime_fpriv->handles);
243 rb_erase(&member->dmabuf_rb, &prime_fpriv->dmabufs);
244
245 dma_buf_put(dma_buf);
246 kfree(member);
247 return;
248 } else if (member->dma_buf < dma_buf) {
249 rb = rb->rb_right;
250 } else {
251 rb = rb->rb_left;
252 }
253 }
254 }
255
drm_gem_map_dma_buf(struct dma_buf_attachment * attach,enum dma_data_direction dir)256 static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
257 enum dma_data_direction dir)
258 {
259 struct drm_prime_attachment *prime_attach = attach->priv;
260 struct drm_gem_object *obj = attach->dmabuf->priv;
261 struct sg_table *sgt;
262
263 if (WARN_ON(dir == DMA_NONE || !prime_attach))
264 return ERR_PTR(-EINVAL);
265
266 /* return the cached mapping when possible */
267 if (prime_attach->dir == dir)
268 return prime_attach->sgt;
269
270 /*
271 * two mappings with different directions for the same attachment are
272 * not allowed
273 */
274 if (WARN_ON(prime_attach->dir != DMA_NONE))
275 return ERR_PTR(-EBUSY);
276
277 sgt = obj->dev->driver->gem_prime_get_sg_table(obj);
278
279 if (!IS_ERR(sgt)) {
280 if (!dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir)) {
281 sg_free_table(sgt);
282 kfree(sgt);
283 sgt = ERR_PTR(-ENOMEM);
284 } else {
285 prime_attach->sgt = sgt;
286 prime_attach->dir = dir;
287 }
288 }
289
290 return sgt;
291 }
292
drm_gem_unmap_dma_buf(struct dma_buf_attachment * attach,struct sg_table * sgt,enum dma_data_direction dir)293 static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
294 struct sg_table *sgt,
295 enum dma_data_direction dir)
296 {
297 /* nothing to be done here */
298 }
299
300 /**
301 * drm_gem_dmabuf_export - dma_buf export implementation for GEM
302 * @dev: parent device for the exported dmabuf
303 * @exp_info: the export information used by dma_buf_export()
304 *
305 * This wraps dma_buf_export() for use by generic GEM drivers that are using
306 * drm_gem_dmabuf_release(). In addition to calling dma_buf_export(), we take
307 * a reference to the &drm_device and the exported &drm_gem_object (stored in
308 * &dma_buf_export_info.priv) which is released by drm_gem_dmabuf_release().
309 *
310 * Returns the new dmabuf.
311 */
drm_gem_dmabuf_export(struct drm_device * dev,struct dma_buf_export_info * exp_info)312 struct dma_buf *drm_gem_dmabuf_export(struct drm_device *dev,
313 struct dma_buf_export_info *exp_info)
314 {
315 struct dma_buf *dma_buf;
316
317 dma_buf = dma_buf_export(exp_info);
318 if (IS_ERR(dma_buf))
319 return dma_buf;
320
321 drm_dev_get(dev);
322 drm_gem_object_get(exp_info->priv);
323
324 return dma_buf;
325 }
326 EXPORT_SYMBOL(drm_gem_dmabuf_export);
327
328 /**
329 * drm_gem_dmabuf_release - dma_buf release implementation for GEM
330 * @dma_buf: buffer to be released
331 *
332 * Generic release function for dma_bufs exported as PRIME buffers. GEM drivers
333 * must use this in their dma_buf ops structure as the release callback.
334 * drm_gem_dmabuf_release() should be used in conjunction with
335 * drm_gem_dmabuf_export().
336 */
drm_gem_dmabuf_release(struct dma_buf * dma_buf)337 void drm_gem_dmabuf_release(struct dma_buf *dma_buf)
338 {
339 struct drm_gem_object *obj = dma_buf->priv;
340 struct drm_device *dev = obj->dev;
341
342 /* drop the reference on the export fd holds */
343 drm_gem_object_put_unlocked(obj);
344
345 drm_dev_put(dev);
346 }
347 EXPORT_SYMBOL(drm_gem_dmabuf_release);
348
drm_gem_dmabuf_vmap(struct dma_buf * dma_buf)349 static void *drm_gem_dmabuf_vmap(struct dma_buf *dma_buf)
350 {
351 struct drm_gem_object *obj = dma_buf->priv;
352 struct drm_device *dev = obj->dev;
353
354 return dev->driver->gem_prime_vmap(obj);
355 }
356
drm_gem_dmabuf_vunmap(struct dma_buf * dma_buf,void * vaddr)357 static void drm_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
358 {
359 struct drm_gem_object *obj = dma_buf->priv;
360 struct drm_device *dev = obj->dev;
361
362 dev->driver->gem_prime_vunmap(obj, vaddr);
363 }
364
drm_gem_dmabuf_kmap_atomic(struct dma_buf * dma_buf,unsigned long page_num)365 static void *drm_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf,
366 unsigned long page_num)
367 {
368 return NULL;
369 }
370
drm_gem_dmabuf_kunmap_atomic(struct dma_buf * dma_buf,unsigned long page_num,void * addr)371 static void drm_gem_dmabuf_kunmap_atomic(struct dma_buf *dma_buf,
372 unsigned long page_num, void *addr)
373 {
374
375 }
drm_gem_dmabuf_kmap(struct dma_buf * dma_buf,unsigned long page_num)376 static void *drm_gem_dmabuf_kmap(struct dma_buf *dma_buf,
377 unsigned long page_num)
378 {
379 return NULL;
380 }
381
drm_gem_dmabuf_kunmap(struct dma_buf * dma_buf,unsigned long page_num,void * addr)382 static void drm_gem_dmabuf_kunmap(struct dma_buf *dma_buf,
383 unsigned long page_num, void *addr)
384 {
385
386 }
387
drm_gem_dmabuf_mmap(struct dma_buf * dma_buf,struct vm_area_struct * vma)388 static int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf,
389 struct vm_area_struct *vma)
390 {
391 struct drm_gem_object *obj = dma_buf->priv;
392 struct drm_device *dev = obj->dev;
393
394 if (!dev->driver->gem_prime_mmap)
395 return -ENOSYS;
396
397 return dev->driver->gem_prime_mmap(obj, vma);
398 }
399
400 static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = {
401 .attach = drm_gem_map_attach,
402 .detach = drm_gem_map_detach,
403 .map_dma_buf = drm_gem_map_dma_buf,
404 .unmap_dma_buf = drm_gem_unmap_dma_buf,
405 .release = drm_gem_dmabuf_release,
406 .map = drm_gem_dmabuf_kmap,
407 .map_atomic = drm_gem_dmabuf_kmap_atomic,
408 .unmap = drm_gem_dmabuf_kunmap,
409 .unmap_atomic = drm_gem_dmabuf_kunmap_atomic,
410 .mmap = drm_gem_dmabuf_mmap,
411 .vmap = drm_gem_dmabuf_vmap,
412 .vunmap = drm_gem_dmabuf_vunmap,
413 };
414
415 /**
416 * DOC: PRIME Helpers
417 *
418 * Drivers can implement @gem_prime_export and @gem_prime_import in terms of
419 * simpler APIs by using the helper functions @drm_gem_prime_export and
420 * @drm_gem_prime_import. These functions implement dma-buf support in terms of
421 * six lower-level driver callbacks:
422 *
423 * Export callbacks:
424 *
425 * * @gem_prime_pin (optional): prepare a GEM object for exporting
426 * * @gem_prime_get_sg_table: provide a scatter/gather table of pinned pages
427 * * @gem_prime_vmap: vmap a buffer exported by your driver
428 * * @gem_prime_vunmap: vunmap a buffer exported by your driver
429 * * @gem_prime_mmap (optional): mmap a buffer exported by your driver
430 *
431 * Import callback:
432 *
433 * * @gem_prime_import_sg_table (import): produce a GEM object from another
434 * driver's scatter/gather table
435 */
436
437 /**
438 * drm_gem_prime_export - helper library implementation of the export callback
439 * @dev: drm_device to export from
440 * @obj: GEM object to export
441 * @flags: flags like DRM_CLOEXEC and DRM_RDWR
442 *
443 * This is the implementation of the gem_prime_export functions for GEM drivers
444 * using the PRIME helpers.
445 */
drm_gem_prime_export(struct drm_device * dev,struct drm_gem_object * obj,int flags)446 struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
447 struct drm_gem_object *obj,
448 int flags)
449 {
450 struct dma_buf_export_info exp_info = {
451 #if 0
452 .exp_name = KBUILD_MODNAME, /* white lie for debug */
453 .owner = dev->driver->fops->owner,
454 #endif
455 .ops = &drm_gem_prime_dmabuf_ops,
456 .size = obj->size,
457 .flags = flags,
458 .priv = obj,
459 };
460
461 if (dev->driver->gem_prime_res_obj)
462 exp_info.resv = dev->driver->gem_prime_res_obj(obj);
463
464 return drm_gem_dmabuf_export(dev, &exp_info);
465 }
466 EXPORT_SYMBOL(drm_gem_prime_export);
467
export_and_register_object(struct drm_device * dev,struct drm_gem_object * obj,uint32_t flags)468 static struct dma_buf *export_and_register_object(struct drm_device *dev,
469 struct drm_gem_object *obj,
470 uint32_t flags)
471 {
472 struct dma_buf *dmabuf;
473
474 /* prevent races with concurrent gem_close. */
475 if (obj->handle_count == 0) {
476 dmabuf = ERR_PTR(-ENOENT);
477 return dmabuf;
478 }
479
480 dmabuf = dev->driver->gem_prime_export(dev, obj, flags);
481 if (IS_ERR(dmabuf)) {
482 /* normally the created dma-buf takes ownership of the ref,
483 * but if that fails then drop the ref
484 */
485 return dmabuf;
486 }
487
488 /*
489 * Note that callers do not need to clean up the export cache
490 * since the check for obj->handle_count guarantees that someone
491 * will clean it up.
492 */
493 obj->dma_buf = dmabuf;
494 get_dma_buf(obj->dma_buf);
495
496 return dmabuf;
497 }
498
499 /**
500 * drm_gem_prime_handle_to_fd - PRIME export function for GEM drivers
501 * @dev: dev to export the buffer from
502 * @file_priv: drm file-private structure
503 * @handle: buffer handle to export
504 * @flags: flags like DRM_CLOEXEC
505 * @prime_fd: pointer to storage for the fd id of the create dma-buf
506 *
507 * This is the PRIME export function which must be used mandatorily by GEM
508 * drivers to ensure correct lifetime management of the underlying GEM object.
509 * The actual exporting from GEM object to a dma-buf is done through the
510 * gem_prime_export driver callback.
511 */
drm_gem_prime_handle_to_fd(struct drm_device * dev,struct drm_file * file_priv,uint32_t handle,uint32_t flags,int * prime_fd)512 int drm_gem_prime_handle_to_fd(struct drm_device *dev,
513 struct drm_file *file_priv, uint32_t handle,
514 uint32_t flags,
515 int *prime_fd)
516 {
517 struct drm_gem_object *obj;
518 int ret = 0;
519 struct dma_buf *dmabuf;
520
521 mutex_lock(&file_priv->prime.lock);
522 obj = drm_gem_object_lookup(file_priv, handle);
523 if (!obj) {
524 ret = -ENOENT;
525 goto out_unlock;
526 }
527
528 dmabuf = drm_prime_lookup_buf_by_handle(&file_priv->prime, handle);
529 if (dmabuf) {
530 get_dma_buf(dmabuf);
531 goto out_have_handle;
532 }
533
534 mutex_lock(&dev->object_name_lock);
535 /* re-export the original imported object */
536 if (obj->import_attach) {
537 dmabuf = obj->import_attach->dmabuf;
538 get_dma_buf(dmabuf);
539 goto out_have_obj;
540 }
541
542 if (obj->dma_buf) {
543 get_dma_buf(obj->dma_buf);
544 dmabuf = obj->dma_buf;
545 goto out_have_obj;
546 }
547
548 dmabuf = export_and_register_object(dev, obj, flags);
549 if (IS_ERR(dmabuf)) {
550 /* normally the created dma-buf takes ownership of the ref,
551 * but if that fails then drop the ref
552 */
553 ret = PTR_ERR(dmabuf);
554 mutex_unlock(&dev->object_name_lock);
555 goto out;
556 }
557
558 out_have_obj:
559 /*
560 * If we've exported this buffer then cheat and add it to the import list
561 * so we get the correct handle back. We must do this under the
562 * protection of dev->object_name_lock to ensure that a racing gem close
563 * ioctl doesn't miss to remove this buffer handle from the cache.
564 */
565 ret = drm_prime_add_buf_handle(&file_priv->prime,
566 dmabuf, handle);
567 mutex_unlock(&dev->object_name_lock);
568 if (ret)
569 goto fail_put_dmabuf;
570
571 out_have_handle:
572 ret = dma_buf_fd(dmabuf, flags);
573 /*
574 * We must _not_ remove the buffer from the handle cache since the newly
575 * created dma buf is already linked in the global obj->dma_buf pointer,
576 * and that is invariant as long as a userspace gem handle exists.
577 * Closing the handle will clean out the cache anyway, so we don't leak.
578 */
579 if (ret < 0) {
580 goto fail_put_dmabuf;
581 } else {
582 *prime_fd = ret;
583 ret = 0;
584 }
585
586 goto out;
587
588 fail_put_dmabuf:
589 dma_buf_put(dmabuf);
590 out:
591 drm_gem_object_put_unlocked(obj);
592 out_unlock:
593 mutex_unlock(&file_priv->prime.lock);
594
595 return ret;
596 }
597 EXPORT_SYMBOL(drm_gem_prime_handle_to_fd);
598
599 /**
600 * drm_gem_prime_import_dev - core implementation of the import callback
601 * @dev: drm_device to import into
602 * @dma_buf: dma-buf object to import
603 * @attach_dev: struct device to dma_buf attach
604 *
605 * This is the core of drm_gem_prime_import. It's designed to be called by
606 * drivers who want to use a different device structure than dev->dev for
607 * attaching via dma_buf.
608 */
drm_gem_prime_import_dev(struct drm_device * dev,struct dma_buf * dma_buf,struct device * attach_dev)609 struct drm_gem_object *drm_gem_prime_import_dev(struct drm_device *dev,
610 struct dma_buf *dma_buf,
611 struct device *attach_dev)
612 {
613 struct dma_buf_attachment *attach;
614 struct sg_table *sgt;
615 struct drm_gem_object *obj;
616 int ret;
617
618 if (dma_buf->ops == &drm_gem_prime_dmabuf_ops) {
619 obj = dma_buf->priv;
620 if (obj->dev == dev) {
621 /*
622 * Importing dmabuf exported from out own gem increases
623 * refcount on gem itself instead of f_count of dmabuf.
624 */
625 drm_gem_object_get(obj);
626 return obj;
627 }
628 }
629
630 if (!dev->driver->gem_prime_import_sg_table)
631 return ERR_PTR(-EINVAL);
632
633 attach = dma_buf_attach(dma_buf, attach_dev);
634 if (IS_ERR(attach))
635 return ERR_CAST(attach);
636
637 get_dma_buf(dma_buf);
638
639 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
640 if (IS_ERR(sgt)) {
641 ret = PTR_ERR(sgt);
642 goto fail_detach;
643 }
644
645 obj = dev->driver->gem_prime_import_sg_table(dev, attach, sgt);
646 if (IS_ERR(obj)) {
647 ret = PTR_ERR(obj);
648 goto fail_unmap;
649 }
650
651 obj->import_attach = attach;
652
653 return obj;
654
655 fail_unmap:
656 dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
657 fail_detach:
658 dma_buf_detach(dma_buf, attach);
659 dma_buf_put(dma_buf);
660
661 return ERR_PTR(ret);
662 }
663 EXPORT_SYMBOL(drm_gem_prime_import_dev);
664
665 /**
666 * drm_gem_prime_import - helper library implementation of the import callback
667 * @dev: drm_device to import into
668 * @dma_buf: dma-buf object to import
669 *
670 * This is the implementation of the gem_prime_import functions for GEM drivers
671 * using the PRIME helpers.
672 */
drm_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)673 struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev,
674 struct dma_buf *dma_buf)
675 {
676 return drm_gem_prime_import_dev(dev, dma_buf, dev->dev);
677 }
678 EXPORT_SYMBOL(drm_gem_prime_import);
679
680 /**
681 * drm_gem_prime_fd_to_handle - PRIME import function for GEM drivers
682 * @dev: dev to export the buffer from
683 * @file_priv: drm file-private structure
684 * @prime_fd: fd id of the dma-buf which should be imported
685 * @handle: pointer to storage for the handle of the imported buffer object
686 *
687 * This is the PRIME import function which must be used mandatorily by GEM
688 * drivers to ensure correct lifetime management of the underlying GEM object.
689 * The actual importing of GEM object from the dma-buf is done through the
690 * gem_import_export driver callback.
691 */
drm_gem_prime_fd_to_handle(struct drm_device * dev,struct drm_file * file_priv,int prime_fd,uint32_t * handle)692 int drm_gem_prime_fd_to_handle(struct drm_device *dev,
693 struct drm_file *file_priv, int prime_fd,
694 uint32_t *handle)
695 {
696 struct dma_buf *dma_buf;
697 struct drm_gem_object *obj;
698 int ret;
699
700 dma_buf = dma_buf_get(prime_fd);
701 if (IS_ERR(dma_buf))
702 return PTR_ERR(dma_buf);
703
704 mutex_lock(&file_priv->prime.lock);
705
706 ret = drm_prime_lookup_buf_handle(&file_priv->prime,
707 dma_buf, handle);
708 if (ret == 0)
709 goto out_put;
710
711 /* never seen this one, need to import */
712 mutex_lock(&dev->object_name_lock);
713 obj = dev->driver->gem_prime_import(dev, dma_buf);
714 if (IS_ERR(obj)) {
715 ret = PTR_ERR(obj);
716 goto out_unlock;
717 }
718
719 if (obj->dma_buf) {
720 WARN_ON(obj->dma_buf != dma_buf);
721 } else {
722 obj->dma_buf = dma_buf;
723 get_dma_buf(dma_buf);
724 }
725
726 /* _handle_create_tail unconditionally unlocks dev->object_name_lock. */
727 ret = drm_gem_handle_create_tail(file_priv, obj, handle);
728 drm_gem_object_put_unlocked(obj);
729 if (ret)
730 goto out_put;
731
732 ret = drm_prime_add_buf_handle(&file_priv->prime,
733 dma_buf, *handle);
734 mutex_unlock(&file_priv->prime.lock);
735 if (ret)
736 goto fail;
737
738 dma_buf_put(dma_buf);
739
740 return 0;
741
742 fail:
743 /* hmm, if driver attached, we are relying on the free-object path
744 * to detach.. which seems ok..
745 */
746 drm_gem_handle_delete(file_priv, *handle);
747 dma_buf_put(dma_buf);
748 return ret;
749
750 out_unlock:
751 mutex_unlock(&dev->object_name_lock);
752 out_put:
753 mutex_unlock(&file_priv->prime.lock);
754 dma_buf_put(dma_buf);
755 return ret;
756 }
757 EXPORT_SYMBOL(drm_gem_prime_fd_to_handle);
758
drm_prime_handle_to_fd_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)759 int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data,
760 struct drm_file *file_priv)
761 {
762 struct drm_prime_handle *args = data;
763
764 if (!drm_core_check_feature(dev, DRIVER_PRIME))
765 return -EINVAL;
766
767 if (!dev->driver->prime_handle_to_fd)
768 return -ENOSYS;
769
770 /* check flags are valid */
771 if (args->flags & ~(DRM_CLOEXEC | DRM_RDWR))
772 return -EINVAL;
773
774 return dev->driver->prime_handle_to_fd(dev, file_priv,
775 args->handle, args->flags, &args->fd);
776 }
777
drm_prime_fd_to_handle_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)778 int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data,
779 struct drm_file *file_priv)
780 {
781 struct drm_prime_handle *args = data;
782
783 if (!drm_core_check_feature(dev, DRIVER_PRIME))
784 return -EINVAL;
785
786 if (!dev->driver->prime_fd_to_handle)
787 return -ENOSYS;
788
789 return dev->driver->prime_fd_to_handle(dev, file_priv,
790 args->fd, &args->handle);
791 }
792
793 /**
794 * drm_prime_pages_to_sg - converts a page array into an sg list
795 * @pages: pointer to the array of page pointers to convert
796 * @nr_pages: length of the page vector
797 *
798 * This helper creates an sg table object from a set of pages
799 * the driver is responsible for mapping the pages into the
800 * importers address space for use with dma_buf itself.
801 */
drm_prime_pages_to_sg(struct page ** pages,unsigned int nr_pages)802 struct sg_table *drm_prime_pages_to_sg(struct page **pages, unsigned int nr_pages)
803 {
804 struct sg_table *sg = NULL;
805 int ret;
806
807 sg = kmalloc(sizeof(struct sg_table), M_DRM, GFP_KERNEL);
808 if (!sg) {
809 ret = -ENOMEM;
810 goto out;
811 }
812
813 ret = sg_alloc_table_from_pages(sg, pages, nr_pages, 0,
814 nr_pages << PAGE_SHIFT, GFP_KERNEL);
815 if (ret)
816 goto out;
817
818 return sg;
819 out:
820 kfree(sg);
821 return ERR_PTR(ret);
822 }
823 EXPORT_SYMBOL(drm_prime_pages_to_sg);
824
825 /**
826 * drm_prime_sg_to_page_addr_arrays - convert an sg table into a page array
827 * @sgt: scatter-gather table to convert
828 * @pages: array of page pointers to store the page array in
829 * @addrs: optional array to store the dma bus address of each page
830 * @max_pages: size of both the passed-in arrays
831 *
832 * Exports an sg table into an array of pages and addresses. This is currently
833 * required by the TTM driver in order to do correct fault handling.
834 */
drm_prime_sg_to_page_addr_arrays(struct sg_table * sgt,struct page ** pages,dma_addr_t * addrs,int max_pages)835 int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages,
836 dma_addr_t *addrs, int max_pages)
837 {
838 unsigned count;
839 struct scatterlist *sg;
840 struct page *page;
841 u32 len;
842 int pg_index;
843 dma_addr_t addr;
844
845 pg_index = 0;
846 for_each_sg(sgt->sgl, sg, sgt->nents, count) {
847 len = sg->length;
848 page = sg_page(sg);
849 addr = sg_dma_address(sg);
850
851 while (len > 0) {
852 if (WARN_ON(pg_index >= max_pages))
853 return -1;
854 pages[pg_index] = page;
855 if (addrs)
856 addrs[pg_index] = addr;
857
858 page++;
859 addr += PAGE_SIZE;
860 len -= PAGE_SIZE;
861 pg_index++;
862 }
863 }
864 return 0;
865 }
866 EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays);
867
868 /**
869 * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object
870 * @obj: GEM object which was created from a dma-buf
871 * @sg: the sg-table which was pinned at import time
872 *
873 * This is the cleanup functions which GEM drivers need to call when they use
874 * @drm_gem_prime_import to import dma-bufs.
875 */
drm_prime_gem_destroy(struct drm_gem_object * obj,struct sg_table * sg)876 void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg)
877 {
878 struct dma_buf_attachment *attach;
879 struct dma_buf *dma_buf;
880 attach = obj->import_attach;
881 if (sg)
882 dma_buf_unmap_attachment(attach, sg, DMA_BIDIRECTIONAL);
883 dma_buf = attach->dmabuf;
884 dma_buf_detach(attach->dmabuf, attach);
885 /* remove the reference */
886 dma_buf_put(dma_buf);
887 }
888 EXPORT_SYMBOL(drm_prime_gem_destroy);
889
drm_prime_init_file_private(struct drm_prime_file_private * prime_fpriv)890 void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv)
891 {
892 lockinit(&prime_fpriv->lock, "drmpfpl", 0, LK_CANRECURSE);
893 prime_fpriv->dmabufs = LINUX_RB_ROOT;
894 prime_fpriv->handles = LINUX_RB_ROOT;
895 }
896
drm_prime_destroy_file_private(struct drm_prime_file_private * prime_fpriv)897 void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv)
898 {
899 /* by now drm_gem_release should've made sure the list is empty */
900 WARN_ON(!RB_EMPTY_ROOT(&prime_fpriv->dmabufs));
901 }
902