1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28 #include <linux/ktime.h>
29 #include <linux/module.h>
30 #include <linux/pagemap.h>
31 #include <linux/pci.h>
32 #include <linux/dma-buf.h>
33
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_exec.h>
37 #include <drm/drm_gem_ttm_helper.h>
38 #include <drm/ttm/ttm_tt.h>
39
40 #include "amdgpu.h"
41 #include "amdgpu_display.h"
42 #include "amdgpu_dma_buf.h"
43 #include "amdgpu_hmm.h"
44 #include "amdgpu_xgmi.h"
45
46 #ifdef __linux__
amdgpu_gem_fault(struct vm_fault * vmf)47 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
48 {
49 struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
50 struct drm_device *ddev = bo->base.dev;
51 vm_fault_t ret;
52 int idx;
53
54 ret = ttm_bo_vm_reserve(bo, vmf);
55 if (ret)
56 return ret;
57
58 if (drm_dev_enter(ddev, &idx)) {
59 ret = amdgpu_bo_fault_reserve_notify(bo);
60 if (ret) {
61 drm_dev_exit(idx);
62 goto unlock;
63 }
64
65 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
66 TTM_BO_VM_NUM_PREFAULT);
67
68 drm_dev_exit(idx);
69 } else {
70 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
71 }
72 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
73 return ret;
74
75 unlock:
76 dma_resv_unlock(bo->base.resv);
77 return ret;
78 }
79
80 static const struct vm_operations_struct amdgpu_gem_vm_ops = {
81 .fault = amdgpu_gem_fault,
82 .open = ttm_bo_vm_open,
83 .close = ttm_bo_vm_close,
84 .access = ttm_bo_vm_access
85 };
86 #else /* !__linux__ */
87 int
amdgpu_gem_fault(struct uvm_faultinfo * ufi,vaddr_t vaddr,vm_page_t * pps,int npages,int centeridx,vm_fault_t fault_type,vm_prot_t access_type,int flags)88 amdgpu_gem_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps,
89 int npages, int centeridx, vm_fault_t fault_type,
90 vm_prot_t access_type, int flags)
91 {
92 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
93 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)uobj;
94 struct drm_device *ddev = bo->base.dev;
95 vm_fault_t ret;
96 int idx;
97
98 ret = ttm_bo_vm_reserve(bo);
99 if (ret) {
100 goto out;
101 }
102
103 if (drm_dev_enter(ddev, &idx)) {
104 ret = amdgpu_bo_fault_reserve_notify(bo);
105 if (ret) {
106 drm_dev_exit(idx);
107 goto unlock;
108 }
109
110 ret = ttm_bo_vm_fault_reserved(ufi, vaddr,
111 TTM_BO_VM_NUM_PREFAULT, 1);
112
113 drm_dev_exit(idx);
114 } else {
115 STUB();
116 #ifdef notyet
117 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
118 #endif
119 }
120 #ifdef __linux__
121 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
122 return ret;
123 #endif
124
125 unlock:
126 dma_resv_unlock(bo->base.resv);
127 out:
128 switch (ret) {
129 case VM_FAULT_NOPAGE:
130 ret = 0;
131 break;
132 case VM_FAULT_RETRY:
133 ret = ERESTART;
134 break;
135 default:
136 ret = EACCES;
137 break;
138 }
139 uvmfault_unlockall(ufi, NULL, uobj);
140 return ret;
141 }
142
143 void
amdgpu_gem_vm_reference(struct uvm_object * uobj)144 amdgpu_gem_vm_reference(struct uvm_object *uobj)
145 {
146 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)uobj;
147
148 ttm_bo_get(bo);
149 }
150
151 void
amdgpu_gem_vm_detach(struct uvm_object * uobj)152 amdgpu_gem_vm_detach(struct uvm_object *uobj)
153 {
154 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)uobj;
155
156 ttm_bo_put(bo);
157 }
158
159 static const struct uvm_pagerops amdgpu_gem_vm_ops = {
160 .pgo_fault = amdgpu_gem_fault,
161 .pgo_reference = amdgpu_gem_vm_reference,
162 .pgo_detach = amdgpu_gem_vm_detach
163 };
164 #endif /* !__linux__ */
165
amdgpu_gem_object_free(struct drm_gem_object * gobj)166 static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
167 {
168 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
169
170 if (aobj) {
171 amdgpu_hmm_unregister(aobj);
172 ttm_bo_put(&aobj->tbo);
173 }
174 }
175
amdgpu_gem_object_create(struct amdgpu_device * adev,unsigned long size,int alignment,u32 initial_domain,u64 flags,enum ttm_bo_type type,struct dma_resv * resv,struct drm_gem_object ** obj,int8_t xcp_id_plus1)176 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
177 int alignment, u32 initial_domain,
178 u64 flags, enum ttm_bo_type type,
179 struct dma_resv *resv,
180 struct drm_gem_object **obj, int8_t xcp_id_plus1)
181 {
182 struct amdgpu_bo *bo;
183 struct amdgpu_bo_user *ubo;
184 struct amdgpu_bo_param bp;
185 int r;
186
187 memset(&bp, 0, sizeof(bp));
188 *obj = NULL;
189 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
190
191 bp.size = size;
192 bp.byte_align = alignment;
193 bp.type = type;
194 bp.resv = resv;
195 bp.preferred_domain = initial_domain;
196 bp.flags = flags;
197 bp.domain = initial_domain;
198 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
199 bp.xcp_id_plus1 = xcp_id_plus1;
200
201 r = amdgpu_bo_create_user(adev, &bp, &ubo);
202 if (r)
203 return r;
204
205 bo = &ubo->bo;
206 *obj = &bo->tbo.base;
207
208 return 0;
209 }
210
211 int drm_file_cmp(struct drm_file *, struct drm_file *);
212 SPLAY_PROTOTYPE(drm_file_tree, drm_file, link, drm_file_cmp);
213
amdgpu_gem_force_release(struct amdgpu_device * adev)214 void amdgpu_gem_force_release(struct amdgpu_device *adev)
215 {
216 struct drm_device *ddev = adev_to_drm(adev);
217 struct drm_file *file;
218
219 mutex_lock(&ddev->filelist_mutex);
220
221 #ifdef __linux__
222 list_for_each_entry(file, &ddev->filelist, lhead) {
223 #else
224 SPLAY_FOREACH(file, drm_file_tree, &ddev->files) {
225 #endif
226 struct drm_gem_object *gobj;
227 int handle;
228
229 WARN_ONCE(1, "Still active user space clients!\n");
230 spin_lock(&file->table_lock);
231 idr_for_each_entry(&file->object_idr, gobj, handle) {
232 WARN_ONCE(1, "And also active allocations!\n");
233 drm_gem_object_put(gobj);
234 }
235 idr_destroy(&file->object_idr);
236 spin_unlock(&file->table_lock);
237 }
238
239 mutex_unlock(&ddev->filelist_mutex);
240 }
241
242 /*
243 * Call from drm_gem_handle_create which appear in both new and open ioctl
244 * case.
245 */
246 static int amdgpu_gem_object_open(struct drm_gem_object *obj,
247 struct drm_file *file_priv)
248 {
249 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
250 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
251 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
252 struct amdgpu_vm *vm = &fpriv->vm;
253 struct amdgpu_bo_va *bo_va;
254 #ifdef notyet
255 struct mm_struct *mm;
256 #endif
257 int r;
258
259 #ifdef notyet
260 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
261 if (mm && mm != current->mm)
262 return -EPERM;
263 #endif
264
265 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
266 !amdgpu_vm_is_bo_always_valid(vm, abo))
267 return -EPERM;
268
269 r = amdgpu_bo_reserve(abo, false);
270 if (r)
271 return r;
272
273 bo_va = amdgpu_vm_bo_find(vm, abo);
274 if (!bo_va)
275 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
276 else
277 ++bo_va->ref_count;
278 amdgpu_bo_unreserve(abo);
279
280 /* Validate and add eviction fence to DMABuf imports with dynamic
281 * attachment in compute VMs. Re-validation will be done by
282 * amdgpu_vm_validate. Fences are on the reservation shared with the
283 * export, which is currently required to be validated and fenced
284 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
285 *
286 * Nested locking below for the case that a GEM object is opened in
287 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
288 * but not for export, this is a different lock class that cannot lead to
289 * circular lock dependencies.
290 */
291 if (!vm->is_compute_context || !vm->process_info)
292 return 0;
293 if (!obj->import_attach ||
294 !dma_buf_is_dynamic(obj->import_attach->dmabuf))
295 return 0;
296 mutex_lock_nested(&vm->process_info->lock, 1);
297 if (!WARN_ON(!vm->process_info->eviction_fence)) {
298 r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,
299 &vm->process_info->eviction_fence->base);
300 if (r) {
301 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
302
303 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
304 if (ti) {
305 dev_warn(adev->dev, "pid %d\n", ti->pid);
306 amdgpu_vm_put_task_info(ti);
307 }
308 }
309 }
310 mutex_unlock(&vm->process_info->lock);
311
312 return r;
313 }
314
315 static void amdgpu_gem_object_close(struct drm_gem_object *obj,
316 struct drm_file *file_priv)
317 {
318 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
319 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
320 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
321 struct amdgpu_vm *vm = &fpriv->vm;
322
323 struct dma_fence *fence = NULL;
324 struct amdgpu_bo_va *bo_va;
325 struct drm_exec exec;
326 long r;
327
328 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
329 drm_exec_until_all_locked(&exec) {
330 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
331 drm_exec_retry_on_contention(&exec);
332 if (unlikely(r))
333 goto out_unlock;
334
335 r = amdgpu_vm_lock_pd(vm, &exec, 0);
336 drm_exec_retry_on_contention(&exec);
337 if (unlikely(r))
338 goto out_unlock;
339 }
340
341 bo_va = amdgpu_vm_bo_find(vm, bo);
342 if (!bo_va || --bo_va->ref_count)
343 goto out_unlock;
344
345 amdgpu_vm_bo_del(adev, bo_va);
346 if (!amdgpu_vm_ready(vm))
347 goto out_unlock;
348
349 r = amdgpu_vm_clear_freed(adev, vm, &fence);
350 if (unlikely(r < 0))
351 dev_err(adev->dev, "failed to clear page "
352 "tables on GEM object close (%ld)\n", r);
353 if (r || !fence)
354 goto out_unlock;
355
356 amdgpu_bo_fence(bo, fence, true);
357 dma_fence_put(fence);
358
359 out_unlock:
360 if (r)
361 dev_err(adev->dev, "leaking bo va (%ld)\n", r);
362 drm_exec_fini(&exec);
363 }
364
365 #ifdef __linux__
366 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
367 {
368 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
369
370 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
371 return -EPERM;
372 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
373 return -EPERM;
374
375 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
376 * for debugger access to invisible VRAM. Should have used MAP_SHARED
377 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
378 * becoming writable and makes is_cow_mapping(vm_flags) false.
379 */
380 if (is_cow_mapping(vma->vm_flags) &&
381 !(vma->vm_flags & VM_ACCESS_FLAGS))
382 vm_flags_clear(vma, VM_MAYWRITE);
383
384 return drm_gem_ttm_mmap(obj, vma);
385 }
386 #else
387 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj,
388 vm_prot_t accessprot, voff_t off, vsize_t size)
389 {
390 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
391
392 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
393 return -EPERM;
394 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
395 return -EPERM;
396
397 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
398 * for debugger access to invisible VRAM. Should have used MAP_SHARED
399 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
400 * becoming writable and makes is_cow_mapping(vm_flags) false.
401 */
402 #ifdef notyet
403 if (is_cow_mapping(vma->vm_flags) &&
404 !(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
405 vma->vm_flags &= ~VM_MAYWRITE;
406 #endif
407
408 return drm_gem_ttm_mmap(obj, accessprot, off, size);
409 }
410 #endif
411
412 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
413 .free = amdgpu_gem_object_free,
414 .open = amdgpu_gem_object_open,
415 .close = amdgpu_gem_object_close,
416 .export = amdgpu_gem_prime_export,
417 .vmap = drm_gem_ttm_vmap,
418 .vunmap = drm_gem_ttm_vunmap,
419 .mmap = amdgpu_gem_object_mmap,
420 .vm_ops = &amdgpu_gem_vm_ops,
421 };
422
423 /*
424 * GEM ioctls.
425 */
426 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
427 struct drm_file *filp)
428 {
429 struct amdgpu_device *adev = drm_to_adev(dev);
430 struct amdgpu_fpriv *fpriv = filp->driver_priv;
431 struct amdgpu_vm *vm = &fpriv->vm;
432 union drm_amdgpu_gem_create *args = data;
433 uint64_t flags = args->in.domain_flags;
434 uint64_t size = args->in.bo_size;
435 struct dma_resv *resv = NULL;
436 struct drm_gem_object *gobj;
437 uint32_t handle, initial_domain;
438 int r;
439
440 /* reject DOORBELLs until userspace code to use it is available */
441 if (args->in.domains & AMDGPU_GEM_DOMAIN_DOORBELL)
442 return -EINVAL;
443
444 /* reject invalid gem flags */
445 if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
446 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
447 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
448 AMDGPU_GEM_CREATE_VRAM_CLEARED |
449 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
450 AMDGPU_GEM_CREATE_EXPLICIT_SYNC |
451 AMDGPU_GEM_CREATE_ENCRYPTED |
452 AMDGPU_GEM_CREATE_GFX12_DCC |
453 AMDGPU_GEM_CREATE_DISCARDABLE))
454 return -EINVAL;
455
456 /* reject invalid gem domains */
457 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
458 return -EINVAL;
459
460 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
461 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
462 return -EINVAL;
463 }
464
465 /* always clear VRAM */
466 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
467
468 /* create a gem object to contain this object in */
469 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
470 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
471 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
472 /* if gds bo is created from user space, it must be
473 * passed to bo list
474 */
475 DRM_ERROR("GDS bo cannot be per-vm-bo\n");
476 return -EINVAL;
477 }
478 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
479 }
480
481 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
482 r = amdgpu_bo_reserve(vm->root.bo, false);
483 if (r)
484 return r;
485
486 resv = vm->root.bo->tbo.base.resv;
487 }
488
489 initial_domain = (u32)(0xffffffff & args->in.domains);
490 retry:
491 r = amdgpu_gem_object_create(adev, size, args->in.alignment,
492 initial_domain,
493 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
494 if (r && r != -ERESTARTSYS) {
495 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
496 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
497 goto retry;
498 }
499
500 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
501 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
502 goto retry;
503 }
504 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
505 size, initial_domain, args->in.alignment, r);
506 }
507
508 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
509 if (!r) {
510 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
511
512 abo->parent = amdgpu_bo_ref(vm->root.bo);
513 }
514 amdgpu_bo_unreserve(vm->root.bo);
515 }
516 if (r)
517 return r;
518
519 r = drm_gem_handle_create(filp, gobj, &handle);
520 /* drop reference from allocate - handle holds it now */
521 drm_gem_object_put(gobj);
522 if (r)
523 return r;
524
525 memset(args, 0, sizeof(*args));
526 args->out.handle = handle;
527 return 0;
528 }
529
530 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
531 struct drm_file *filp)
532 {
533 return -ENOSYS;
534 #ifdef notyet
535 struct ttm_operation_ctx ctx = { true, false };
536 struct amdgpu_device *adev = drm_to_adev(dev);
537 struct drm_amdgpu_gem_userptr *args = data;
538 struct amdgpu_fpriv *fpriv = filp->driver_priv;
539 struct drm_gem_object *gobj;
540 struct hmm_range *range;
541 struct amdgpu_bo *bo;
542 uint32_t handle;
543 int r;
544
545 args->addr = untagged_addr(args->addr);
546
547 if (offset_in_page(args->addr | args->size))
548 return -EINVAL;
549
550 /* reject unknown flag values */
551 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
552 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
553 AMDGPU_GEM_USERPTR_REGISTER))
554 return -EINVAL;
555
556 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
557 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
558
559 /* if we want to write to it we must install a MMU notifier */
560 return -EACCES;
561 }
562
563 /* create a gem object to contain this object in */
564 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
565 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
566 if (r)
567 return r;
568
569 bo = gem_to_amdgpu_bo(gobj);
570 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
571 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
572 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
573 if (r)
574 goto release_object;
575
576 r = amdgpu_hmm_register(bo, args->addr);
577 if (r)
578 goto release_object;
579
580 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
581 r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
582 &range);
583 if (r)
584 goto release_object;
585
586 r = amdgpu_bo_reserve(bo, true);
587 if (r)
588 goto user_pages_done;
589
590 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
591 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
592 amdgpu_bo_unreserve(bo);
593 if (r)
594 goto user_pages_done;
595 }
596
597 r = drm_gem_handle_create(filp, gobj, &handle);
598 if (r)
599 goto user_pages_done;
600
601 args->handle = handle;
602
603 user_pages_done:
604 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
605 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
606
607 release_object:
608 drm_gem_object_put(gobj);
609
610 return r;
611 #endif
612 }
613
614 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
615 struct drm_device *dev,
616 uint32_t handle, uint64_t *offset_p)
617 {
618 struct drm_gem_object *gobj;
619 struct amdgpu_bo *robj;
620
621 gobj = drm_gem_object_lookup(filp, handle);
622 if (!gobj)
623 return -ENOENT;
624
625 robj = gem_to_amdgpu_bo(gobj);
626 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
627 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
628 drm_gem_object_put(gobj);
629 return -EPERM;
630 }
631 *offset_p = amdgpu_bo_mmap_offset(robj);
632 drm_gem_object_put(gobj);
633 return 0;
634 }
635
636 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
637 struct drm_file *filp)
638 {
639 union drm_amdgpu_gem_mmap *args = data;
640 uint32_t handle = args->in.handle;
641
642 memset(args, 0, sizeof(*args));
643 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
644 }
645
646 /**
647 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
648 *
649 * @timeout_ns: timeout in ns
650 *
651 * Calculate the timeout in jiffies from an absolute timeout in ns.
652 */
653 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
654 {
655 unsigned long timeout_jiffies;
656 ktime_t timeout;
657
658 /* clamp timeout if it's to large */
659 if (((int64_t)timeout_ns) < 0)
660 return MAX_SCHEDULE_TIMEOUT;
661
662 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
663 if (ktime_to_ns(timeout) < 0)
664 return 0;
665
666 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
667 /* clamp timeout to avoid unsigned-> signed overflow */
668 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)
669 return MAX_SCHEDULE_TIMEOUT - 1;
670
671 return timeout_jiffies;
672 }
673
674 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
675 struct drm_file *filp)
676 {
677 union drm_amdgpu_gem_wait_idle *args = data;
678 struct drm_gem_object *gobj;
679 struct amdgpu_bo *robj;
680 uint32_t handle = args->in.handle;
681 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
682 int r = 0;
683 long ret;
684
685 gobj = drm_gem_object_lookup(filp, handle);
686 if (!gobj)
687 return -ENOENT;
688
689 robj = gem_to_amdgpu_bo(gobj);
690 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
691 true, timeout);
692
693 /* ret == 0 means not signaled,
694 * ret > 0 means signaled
695 * ret < 0 means interrupted before timeout
696 */
697 if (ret >= 0) {
698 memset(args, 0, sizeof(*args));
699 args->out.status = (ret == 0);
700 } else
701 r = ret;
702
703 drm_gem_object_put(gobj);
704 return r;
705 }
706
707 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
708 struct drm_file *filp)
709 {
710 struct drm_amdgpu_gem_metadata *args = data;
711 struct drm_gem_object *gobj;
712 struct amdgpu_bo *robj;
713 int r = -1;
714
715 DRM_DEBUG("%d\n", args->handle);
716 gobj = drm_gem_object_lookup(filp, args->handle);
717 if (gobj == NULL)
718 return -ENOENT;
719 robj = gem_to_amdgpu_bo(gobj);
720
721 r = amdgpu_bo_reserve(robj, false);
722 if (unlikely(r != 0))
723 goto out;
724
725 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
726 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
727 r = amdgpu_bo_get_metadata(robj, args->data.data,
728 sizeof(args->data.data),
729 &args->data.data_size_bytes,
730 &args->data.flags);
731 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
732 if (args->data.data_size_bytes > sizeof(args->data.data)) {
733 r = -EINVAL;
734 goto unreserve;
735 }
736 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
737 if (!r)
738 r = amdgpu_bo_set_metadata(robj, args->data.data,
739 args->data.data_size_bytes,
740 args->data.flags);
741 }
742
743 unreserve:
744 amdgpu_bo_unreserve(robj);
745 out:
746 drm_gem_object_put(gobj);
747 return r;
748 }
749
750 /**
751 * amdgpu_gem_va_update_vm -update the bo_va in its VM
752 *
753 * @adev: amdgpu_device pointer
754 * @vm: vm to update
755 * @bo_va: bo_va to update
756 * @operation: map, unmap or clear
757 *
758 * Update the bo_va directly after setting its address. Errors are not
759 * vital here, so they are not reported back to userspace.
760 */
761 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
762 struct amdgpu_vm *vm,
763 struct amdgpu_bo_va *bo_va,
764 uint32_t operation)
765 {
766 int r;
767
768 if (!amdgpu_vm_ready(vm))
769 return;
770
771 r = amdgpu_vm_clear_freed(adev, vm, NULL);
772 if (r)
773 goto error;
774
775 if (operation == AMDGPU_VA_OP_MAP ||
776 operation == AMDGPU_VA_OP_REPLACE) {
777 r = amdgpu_vm_bo_update(adev, bo_va, false);
778 if (r)
779 goto error;
780 }
781
782 r = amdgpu_vm_update_pdes(adev, vm, false);
783
784 error:
785 if (r && r != -ERESTARTSYS)
786 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
787 }
788
789 /**
790 * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
791 *
792 * @adev: amdgpu_device pointer
793 * @flags: GEM UAPI flags
794 *
795 * Returns the GEM UAPI flags mapped into hardware for the ASIC.
796 */
797 uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
798 {
799 uint64_t pte_flag = 0;
800
801 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
802 pte_flag |= AMDGPU_PTE_EXECUTABLE;
803 if (flags & AMDGPU_VM_PAGE_READABLE)
804 pte_flag |= AMDGPU_PTE_READABLE;
805 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
806 pte_flag |= AMDGPU_PTE_WRITEABLE;
807 if (flags & AMDGPU_VM_PAGE_PRT)
808 pte_flag |= AMDGPU_PTE_PRT_FLAG(adev);
809 if (flags & AMDGPU_VM_PAGE_NOALLOC)
810 pte_flag |= AMDGPU_PTE_NOALLOC;
811
812 if (adev->gmc.gmc_funcs->map_mtype)
813 pte_flag |= amdgpu_gmc_map_mtype(adev,
814 flags & AMDGPU_VM_MTYPE_MASK);
815
816 return pte_flag;
817 }
818
819 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
820 struct drm_file *filp)
821 {
822 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
823 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
824 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
825 AMDGPU_VM_PAGE_NOALLOC;
826 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
827 AMDGPU_VM_PAGE_PRT;
828
829 struct drm_amdgpu_gem_va *args = data;
830 struct drm_gem_object *gobj;
831 struct amdgpu_device *adev = drm_to_adev(dev);
832 struct amdgpu_fpriv *fpriv = filp->driver_priv;
833 struct amdgpu_bo *abo;
834 struct amdgpu_bo_va *bo_va;
835 struct drm_exec exec;
836 uint64_t va_flags;
837 uint64_t vm_size;
838 int r = 0;
839
840 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
841 dev_dbg(dev->dev,
842 "va_address 0x%llx is in reserved area 0x%llx\n",
843 args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
844 return -EINVAL;
845 }
846
847 if (args->va_address >= AMDGPU_GMC_HOLE_START &&
848 args->va_address < AMDGPU_GMC_HOLE_END) {
849 dev_dbg(dev->dev,
850 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
851 args->va_address, AMDGPU_GMC_HOLE_START,
852 AMDGPU_GMC_HOLE_END);
853 return -EINVAL;
854 }
855
856 args->va_address &= AMDGPU_GMC_HOLE_MASK;
857
858 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
859 vm_size -= AMDGPU_VA_RESERVED_TOP;
860 if (args->va_address + args->map_size > vm_size) {
861 dev_dbg(dev->dev,
862 "va_address 0x%llx is in top reserved area 0x%llx\n",
863 args->va_address + args->map_size, vm_size);
864 return -EINVAL;
865 }
866
867 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
868 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
869 args->flags);
870 return -EINVAL;
871 }
872
873 switch (args->operation) {
874 case AMDGPU_VA_OP_MAP:
875 case AMDGPU_VA_OP_UNMAP:
876 case AMDGPU_VA_OP_CLEAR:
877 case AMDGPU_VA_OP_REPLACE:
878 break;
879 default:
880 dev_dbg(dev->dev, "unsupported operation %d\n",
881 args->operation);
882 return -EINVAL;
883 }
884
885 if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
886 !(args->flags & AMDGPU_VM_PAGE_PRT)) {
887 gobj = drm_gem_object_lookup(filp, args->handle);
888 if (gobj == NULL)
889 return -ENOENT;
890 abo = gem_to_amdgpu_bo(gobj);
891 } else {
892 gobj = NULL;
893 abo = NULL;
894 }
895
896 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
897 DRM_EXEC_IGNORE_DUPLICATES, 0);
898 drm_exec_until_all_locked(&exec) {
899 if (gobj) {
900 r = drm_exec_lock_obj(&exec, gobj);
901 drm_exec_retry_on_contention(&exec);
902 if (unlikely(r))
903 goto error;
904 }
905
906 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
907 drm_exec_retry_on_contention(&exec);
908 if (unlikely(r))
909 goto error;
910 }
911
912 if (abo) {
913 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
914 if (!bo_va) {
915 r = -ENOENT;
916 goto error;
917 }
918 } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
919 bo_va = fpriv->prt_va;
920 } else {
921 bo_va = NULL;
922 }
923
924 switch (args->operation) {
925 case AMDGPU_VA_OP_MAP:
926 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
927 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
928 args->offset_in_bo, args->map_size,
929 va_flags);
930 break;
931 case AMDGPU_VA_OP_UNMAP:
932 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
933 break;
934
935 case AMDGPU_VA_OP_CLEAR:
936 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
937 args->va_address,
938 args->map_size);
939 break;
940 case AMDGPU_VA_OP_REPLACE:
941 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
942 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
943 args->offset_in_bo, args->map_size,
944 va_flags);
945 break;
946 default:
947 break;
948 }
949 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm)
950 amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
951 args->operation);
952
953 error:
954 drm_exec_fini(&exec);
955 drm_gem_object_put(gobj);
956 return r;
957 }
958
959 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
960 struct drm_file *filp)
961 {
962 struct amdgpu_device *adev = drm_to_adev(dev);
963 struct drm_amdgpu_gem_op *args = data;
964 struct drm_gem_object *gobj;
965 struct amdgpu_vm_bo_base *base;
966 struct amdgpu_bo *robj;
967 int r;
968
969 gobj = drm_gem_object_lookup(filp, args->handle);
970 if (!gobj)
971 return -ENOENT;
972
973 robj = gem_to_amdgpu_bo(gobj);
974
975 r = amdgpu_bo_reserve(robj, false);
976 if (unlikely(r))
977 goto out;
978
979 switch (args->op) {
980 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
981 struct drm_amdgpu_gem_create_in info;
982 void __user *out = u64_to_user_ptr(args->value);
983
984 info.bo_size = robj->tbo.base.size;
985 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
986 info.domains = robj->preferred_domains;
987 info.domain_flags = robj->flags;
988 amdgpu_bo_unreserve(robj);
989 if (copy_to_user(out, &info, sizeof(info)))
990 r = -EFAULT;
991 break;
992 }
993 case AMDGPU_GEM_OP_SET_PLACEMENT:
994 if (robj->tbo.base.import_attach &&
995 args->value & AMDGPU_GEM_DOMAIN_VRAM) {
996 r = -EINVAL;
997 amdgpu_bo_unreserve(robj);
998 break;
999 }
1000 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1001 r = -EPERM;
1002 amdgpu_bo_unreserve(robj);
1003 break;
1004 }
1005 for (base = robj->vm_bo; base; base = base->next)
1006 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1007 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1008 r = -EINVAL;
1009 amdgpu_bo_unreserve(robj);
1010 goto out;
1011 }
1012
1013
1014 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1015 AMDGPU_GEM_DOMAIN_GTT |
1016 AMDGPU_GEM_DOMAIN_CPU);
1017 robj->allowed_domains = robj->preferred_domains;
1018 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1019 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1020
1021 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1022 amdgpu_vm_bo_invalidate(adev, robj, true);
1023
1024 amdgpu_bo_unreserve(robj);
1025 break;
1026 default:
1027 amdgpu_bo_unreserve(robj);
1028 r = -EINVAL;
1029 }
1030
1031 out:
1032 drm_gem_object_put(gobj);
1033 return r;
1034 }
1035
1036 static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1037 int width,
1038 int cpp,
1039 bool tiled)
1040 {
1041 int aligned = width;
1042 int pitch_mask = 0;
1043
1044 switch (cpp) {
1045 case 1:
1046 pitch_mask = 255;
1047 break;
1048 case 2:
1049 pitch_mask = 127;
1050 break;
1051 case 3:
1052 case 4:
1053 pitch_mask = 63;
1054 break;
1055 }
1056
1057 aligned += pitch_mask;
1058 aligned &= ~pitch_mask;
1059 return aligned * cpp;
1060 }
1061
1062 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1063 struct drm_device *dev,
1064 struct drm_mode_create_dumb *args)
1065 {
1066 struct amdgpu_device *adev = drm_to_adev(dev);
1067 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1068 struct drm_gem_object *gobj;
1069 uint32_t handle;
1070 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1071 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1072 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1073 u32 domain;
1074 int r;
1075
1076 /*
1077 * The buffer returned from this function should be cleared, but
1078 * it can only be done if the ring is enabled or we'll fail to
1079 * create the buffer.
1080 */
1081 if (adev->mman.buffer_funcs_enabled)
1082 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1083
1084 args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1085 DIV_ROUND_UP(args->bpp, 8), 0);
1086 args->size = (u64)args->pitch * args->height;
1087 args->size = ALIGN(args->size, PAGE_SIZE);
1088 domain = amdgpu_bo_get_preferred_domain(adev,
1089 amdgpu_display_supported_domains(adev, flags));
1090 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1091 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1092 if (r)
1093 return -ENOMEM;
1094
1095 r = drm_gem_handle_create(file_priv, gobj, &handle);
1096 /* drop reference from allocate - handle holds it now */
1097 drm_gem_object_put(gobj);
1098 if (r)
1099 return r;
1100
1101 args->handle = handle;
1102 return 0;
1103 }
1104
1105 #if defined(CONFIG_DEBUG_FS)
1106 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1107 {
1108 struct amdgpu_device *adev = m->private;
1109 struct drm_device *dev = adev_to_drm(adev);
1110 struct drm_file *file;
1111 int r;
1112
1113 r = mutex_lock_interruptible(&dev->filelist_mutex);
1114 if (r)
1115 return r;
1116
1117 list_for_each_entry(file, &dev->filelist, lhead) {
1118 struct task_struct *task;
1119 struct drm_gem_object *gobj;
1120 struct pid *pid;
1121 int id;
1122
1123 /*
1124 * Although we have a valid reference on file->pid, that does
1125 * not guarantee that the task_struct who called get_pid() is
1126 * still alive (e.g. get_pid(current) => fork() => exit()).
1127 * Therefore, we need to protect this ->comm access using RCU.
1128 */
1129 rcu_read_lock();
1130 pid = rcu_dereference(file->pid);
1131 task = pid_task(pid, PIDTYPE_TGID);
1132 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1133 task ? task->comm : "<unknown>");
1134 rcu_read_unlock();
1135
1136 spin_lock(&file->table_lock);
1137 idr_for_each_entry(&file->object_idr, gobj, id) {
1138 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1139
1140 amdgpu_bo_print_info(id, bo, m);
1141 }
1142 spin_unlock(&file->table_lock);
1143 }
1144
1145 mutex_unlock(&dev->filelist_mutex);
1146 return 0;
1147 }
1148
1149 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1150
1151 #endif
1152
1153 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1154 {
1155 #if defined(CONFIG_DEBUG_FS)
1156 struct drm_minor *minor = adev_to_drm(adev)->primary;
1157 struct dentry *root = minor->debugfs_root;
1158
1159 debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1160 &amdgpu_debugfs_gem_info_fops);
1161 #endif
1162 }
1163