1 /* $NetBSD: ttm_bo.c,v 1.31 2022/02/14 09:25:39 riastradh Exp $ */
2
3 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
4 /**************************************************************************
5 *
6 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
7 * All Rights Reserved.
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the
11 * "Software"), to deal in the Software without restriction, including
12 * without limitation the rights to use, copy, modify, merge, publish,
13 * distribute, sub license, and/or sell copies of the Software, and to
14 * permit persons to whom the Software is furnished to do so, subject to
15 * the following conditions:
16 *
17 * The above copyright notice and this permission notice (including the
18 * next paragraph) shall be included in all copies or substantial portions
19 * of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
24 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
25 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
26 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
27 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: ttm_bo.c,v 1.31 2022/02/14 09:25:39 riastradh Exp $");
36
37 #define pr_fmt(fmt) "[TTM] " fmt
38
39 #ifdef __NetBSD__
40 #include <sys/types.h>
41 #include <uvm/uvm_extern.h>
42 #include <uvm/uvm_object.h>
43 #endif
44
45 #include <drm/drm_prime.h>
46 #include <drm/ttm/ttm_module.h>
47 #include <drm/ttm/ttm_bo_driver.h>
48 #include <drm/ttm/ttm_placement.h>
49 #include <linux/jiffies.h>
50 #include <linux/slab.h>
51 #include <linux/sched.h>
52 #include <linux/mm.h>
53 #include <linux/file.h>
54 #include <linux/module.h>
55 #include <linux/atomic.h>
56 #include <linux/dma-resv.h>
57
58 #include <linux/nbsd-namespace.h>
59
60 #ifndef __NetBSD__ /* XXX sysfs */
61 static void ttm_bo_global_kobj_release(struct kobject *kobj);
62 #endif
63
64 /**
65 * ttm_global_mutex - protecting the global BO state
66 */
67 #ifdef __NetBSD__
68 static struct mutex ttm_global_mutex;
69 unsigned ttm_bo_glob_use_count;
70 struct ttm_bo_global ttm_bo_glob;
71 #else
72 DEFINE_MUTEX(ttm_global_mutex);
73 unsigned ttm_bo_glob_use_count;
74 struct ttm_bo_global ttm_bo_glob;
75 EXPORT_SYMBOL(ttm_bo_glob);
76 #endif
77
78 #ifndef __NetBSD__ /* XXX sysfs */
79 static struct attribute ttm_bo_count = {
80 .name = "bo_count",
81 .mode = S_IRUGO
82 };
83 #endif
84
85 /* default destructor */
ttm_bo_default_destroy(struct ttm_buffer_object * bo)86 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
87 {
88 kfree(bo);
89 }
90
ttm_mem_type_from_place(const struct ttm_place * place,uint32_t * mem_type)91 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
92 uint32_t *mem_type)
93 {
94 int pos;
95
96 pos = ffs(place->flags & TTM_PL_MASK_MEM);
97 if (unlikely(!pos))
98 return -EINVAL;
99
100 *mem_type = pos - 1;
101 return 0;
102 }
103
ttm_mem_type_debug(struct ttm_bo_device * bdev,struct drm_printer * p,int mem_type)104 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,
105 int mem_type)
106 {
107 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
108
109 drm_printf(p, " has_type: %d\n", man->has_type);
110 drm_printf(p, " use_type: %d\n", man->use_type);
111 drm_printf(p, " flags: 0x%08X\n", man->flags);
112 drm_printf(p, " gpu_offset: 0x%08"PRIX64"\n", man->gpu_offset);
113 drm_printf(p, " size: %"PRIu64"\n", man->size);
114 drm_printf(p, " available_caching: 0x%08X\n", man->available_caching);
115 drm_printf(p, " default_caching: 0x%08X\n", man->default_caching);
116 if (mem_type != TTM_PL_SYSTEM)
117 (*man->func->debug)(man, p);
118 }
119
ttm_bo_mem_space_debug(struct ttm_buffer_object * bo,struct ttm_placement * placement)120 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
121 struct ttm_placement *placement)
122 {
123 struct drm_printer p = drm_debug_printer(TTM_PFX);
124 int i, ret, mem_type;
125
126 drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
127 bo, bo->mem.num_pages, bo->mem.size >> 10,
128 bo->mem.size >> 20);
129 for (i = 0; i < placement->num_placement; i++) {
130 ret = ttm_mem_type_from_place(&placement->placement[i],
131 &mem_type);
132 if (ret)
133 return;
134 drm_printf(&p, " placement[%d]=0x%08X (%d)\n",
135 i, placement->placement[i].flags, mem_type);
136 ttm_mem_type_debug(bo->bdev, &p, mem_type);
137 }
138 }
139
140 #ifndef __NetBSD__ /* XXX sysfs */
ttm_bo_global_show(struct kobject * kobj,struct attribute * attr,char * buffer)141 static ssize_t ttm_bo_global_show(struct kobject *kobj,
142 struct attribute *attr,
143 char *buffer)
144 {
145 struct ttm_bo_global *glob =
146 container_of(kobj, struct ttm_bo_global, kobj);
147
148 return snprintf(buffer, PAGE_SIZE, "%d\n",
149 atomic_read(&glob->bo_count));
150 }
151
152 static struct attribute *ttm_bo_global_attrs[] = {
153 &ttm_bo_count,
154 NULL
155 };
156
157 static const struct sysfs_ops ttm_bo_global_ops = {
158 .show = &ttm_bo_global_show
159 };
160
161 static struct kobj_type ttm_bo_glob_kobj_type = {
162 .release = &ttm_bo_global_kobj_release,
163 .sysfs_ops = &ttm_bo_global_ops,
164 .default_attrs = ttm_bo_global_attrs
165 };
166 #endif /* __NetBSD__ */
167
168
ttm_bo_type_flags(unsigned type)169 static inline uint32_t ttm_bo_type_flags(unsigned type)
170 {
171 return 1 << (type);
172 }
173
ttm_bo_release_list(struct kref * list_kref)174 static void ttm_bo_release_list(struct kref *list_kref)
175 {
176 struct ttm_buffer_object *bo =
177 container_of(list_kref, struct ttm_buffer_object, list_kref);
178 size_t acc_size = bo->acc_size;
179
180 BUG_ON(kref_read(&bo->list_kref));
181 BUG_ON(kref_read(&bo->kref));
182 BUG_ON(bo->mem.mm_node != NULL);
183 BUG_ON(!list_empty(&bo->lru));
184 BUG_ON(!list_empty(&bo->ddestroy));
185 ttm_tt_destroy(bo->ttm);
186 atomic_dec(&ttm_bo_glob.bo_count);
187 dma_fence_put(bo->moving);
188 if (!ttm_bo_uses_embedded_gem_object(bo))
189 dma_resv_fini(&bo->base._resv);
190 bo->destroy(bo);
191 ttm_mem_global_free(&ttm_mem_glob, acc_size);
192 }
193
ttm_bo_add_mem_to_lru(struct ttm_buffer_object * bo,struct ttm_mem_reg * mem)194 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
195 struct ttm_mem_reg *mem)
196 {
197 struct ttm_bo_device *bdev = bo->bdev;
198 struct ttm_mem_type_manager *man;
199
200 dma_resv_assert_held(bo->base.resv);
201
202 if (!list_empty(&bo->lru))
203 return;
204
205 if (mem->placement & TTM_PL_FLAG_NO_EVICT)
206 return;
207
208 man = &bdev->man[mem->mem_type];
209 list_add_tail(&bo->lru, &man->lru[bo->priority]);
210 kref_get(&bo->list_kref);
211
212 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm &&
213 !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
214 TTM_PAGE_FLAG_SWAPPED))) {
215 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
216 kref_get(&bo->list_kref);
217 }
218 }
219
ttm_bo_ref_bug(struct kref * list_kref)220 static void ttm_bo_ref_bug(struct kref *list_kref)
221 {
222 BUG();
223 }
224
ttm_bo_del_from_lru(struct ttm_buffer_object * bo)225 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
226 {
227 struct ttm_bo_device *bdev = bo->bdev;
228 bool notify = false;
229
230 if (!list_empty(&bo->swap)) {
231 list_del_init(&bo->swap);
232 kref_put(&bo->list_kref, ttm_bo_ref_bug);
233 notify = true;
234 }
235 if (!list_empty(&bo->lru)) {
236 list_del_init(&bo->lru);
237 kref_put(&bo->list_kref, ttm_bo_ref_bug);
238 notify = true;
239 }
240
241 if (notify && bdev->driver->del_from_lru_notify)
242 bdev->driver->del_from_lru_notify(bo);
243 }
244
ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos * pos,struct ttm_buffer_object * bo)245 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
246 struct ttm_buffer_object *bo)
247 {
248 if (!pos->first)
249 pos->first = bo;
250 pos->last = bo;
251 }
252
ttm_bo_move_to_lru_tail(struct ttm_buffer_object * bo,struct ttm_lru_bulk_move * bulk)253 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
254 struct ttm_lru_bulk_move *bulk)
255 {
256 dma_resv_assert_held(bo->base.resv);
257
258 ttm_bo_del_from_lru(bo);
259 ttm_bo_add_mem_to_lru(bo, &bo->mem);
260
261 if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
262 switch (bo->mem.mem_type) {
263 case TTM_PL_TT:
264 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
265 break;
266
267 case TTM_PL_VRAM:
268 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
269 break;
270 }
271 if (bo->ttm && !(bo->ttm->page_flags &
272 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
273 ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
274 }
275 }
276 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
277
ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move * bulk)278 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
279 {
280 unsigned i;
281
282 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
283 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
284 struct ttm_mem_type_manager *man;
285
286 if (!pos->first)
287 continue;
288
289 dma_resv_assert_held(pos->first->base.resv);
290 dma_resv_assert_held(pos->last->base.resv);
291
292 man = &pos->first->bdev->man[TTM_PL_TT];
293 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
294 &pos->last->lru);
295 }
296
297 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
298 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
299 struct ttm_mem_type_manager *man;
300
301 if (!pos->first)
302 continue;
303
304 dma_resv_assert_held(pos->first->base.resv);
305 dma_resv_assert_held(pos->last->base.resv);
306
307 man = &pos->first->bdev->man[TTM_PL_VRAM];
308 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
309 &pos->last->lru);
310 }
311
312 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
313 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
314 struct list_head *lru;
315
316 if (!pos->first)
317 continue;
318
319 dma_resv_assert_held(pos->first->base.resv);
320 dma_resv_assert_held(pos->last->base.resv);
321
322 lru = &ttm_bo_glob.swap_lru[i];
323 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
324 }
325 }
326 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
327
ttm_bo_handle_move_mem(struct ttm_buffer_object * bo,struct ttm_mem_reg * mem,bool evict,struct ttm_operation_ctx * ctx)328 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
329 struct ttm_mem_reg *mem, bool evict,
330 struct ttm_operation_ctx *ctx)
331 {
332 struct ttm_bo_device *bdev = bo->bdev;
333 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
334 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
335 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
336 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
337 int ret = 0;
338
339 if (old_is_pci || new_is_pci ||
340 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
341 ret = ttm_mem_io_lock(old_man, true);
342 if (unlikely(ret != 0))
343 goto out_err;
344 ttm_bo_unmap_virtual_locked(bo);
345 ttm_mem_io_unlock(old_man);
346 }
347
348 /*
349 * Create and bind a ttm if required.
350 */
351
352 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
353 if (bo->ttm == NULL) {
354 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
355 ret = ttm_tt_create(bo, zero);
356 if (ret)
357 goto out_err;
358 }
359
360 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
361 if (ret)
362 goto out_err;
363
364 if (mem->mem_type != TTM_PL_SYSTEM) {
365 ret = ttm_tt_bind(bo->ttm, mem, ctx);
366 if (ret)
367 goto out_err;
368 }
369
370 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
371 if (bdev->driver->move_notify)
372 bdev->driver->move_notify(bo, evict, mem);
373 bo->mem = *mem;
374 mem->mm_node = NULL;
375 goto moved;
376 }
377 }
378
379 if (bdev->driver->move_notify)
380 bdev->driver->move_notify(bo, evict, mem);
381
382 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
383 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
384 ret = ttm_bo_move_ttm(bo, ctx, mem);
385 else if (bdev->driver->move)
386 ret = bdev->driver->move(bo, evict, ctx, mem);
387 else
388 ret = ttm_bo_move_memcpy(bo, ctx, mem);
389
390 if (ret) {
391 if (bdev->driver->move_notify) {
392 swap(*mem, bo->mem);
393 bdev->driver->move_notify(bo, false, mem);
394 swap(*mem, bo->mem);
395 }
396
397 goto out_err;
398 }
399
400 moved:
401 if (bo->evicted) {
402 if (bdev->driver->invalidate_caches) {
403 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
404 if (ret)
405 pr_err("Can not flush read caches\n");
406 }
407 bo->evicted = false;
408 }
409
410 if (bo->mem.mm_node)
411 bo->offset = (bo->mem.start << PAGE_SHIFT) +
412 bdev->man[bo->mem.mem_type].gpu_offset;
413 else
414 bo->offset = 0;
415
416 ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
417 return 0;
418
419 out_err:
420 new_man = &bdev->man[bo->mem.mem_type];
421 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
422 ttm_tt_destroy(bo->ttm);
423 bo->ttm = NULL;
424 }
425
426 return ret;
427 }
428
429 /**
430 * Call bo::reserved.
431 * Will release GPU memory type usage on destruction.
432 * This is the place to put in driver specific hooks to release
433 * driver private resources.
434 * Will release the bo::reserved lock.
435 */
436
ttm_bo_cleanup_memtype_use(struct ttm_buffer_object * bo)437 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
438 {
439 if (bo->bdev->driver->move_notify)
440 bo->bdev->driver->move_notify(bo, false, NULL);
441
442 ttm_tt_destroy(bo->ttm);
443 bo->ttm = NULL;
444 ttm_bo_mem_put(bo, &bo->mem);
445 }
446
ttm_bo_individualize_resv(struct ttm_buffer_object * bo)447 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
448 {
449 int r;
450
451 if (bo->base.resv == &bo->base._resv)
452 return 0;
453
454 BUG_ON(!dma_resv_trylock(&bo->base._resv));
455
456 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
457 if (r)
458 dma_resv_unlock(&bo->base._resv);
459
460 return r;
461 }
462
ttm_bo_flush_all_fences(struct ttm_buffer_object * bo)463 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
464 {
465 struct dma_resv_list *fobj;
466 struct dma_fence *fence;
467 int i;
468
469 fobj = dma_resv_get_list(&bo->base._resv);
470 fence = dma_resv_get_excl(&bo->base._resv);
471 if (fence && !fence->ops->signaled)
472 dma_fence_enable_sw_signaling(fence);
473
474 for (i = 0; fobj && i < fobj->shared_count; ++i) {
475 fence = rcu_dereference_protected(fobj->shared[i],
476 dma_resv_held(bo->base.resv));
477
478 if (!fence->ops->signaled)
479 dma_fence_enable_sw_signaling(fence);
480 }
481 }
482
ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object * bo)483 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
484 {
485 struct ttm_bo_device *bdev = bo->bdev;
486 int ret;
487
488 ret = ttm_bo_individualize_resv(bo);
489 if (ret) {
490 /* Last resort, if we fail to allocate memory for the
491 * fences block for the BO to become idle
492 */
493 dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
494 30 * HZ);
495 spin_lock(&ttm_bo_glob.lru_lock);
496 goto error;
497 }
498
499 spin_lock(&ttm_bo_glob.lru_lock);
500 ret = dma_resv_trylock(bo->base.resv) ? 0 : -EBUSY;
501 if (!ret) {
502 if (dma_resv_test_signaled_rcu(&bo->base._resv, true)) {
503 ttm_bo_del_from_lru(bo);
504 spin_unlock(&ttm_bo_glob.lru_lock);
505 if (bo->base.resv != &bo->base._resv)
506 dma_resv_unlock(&bo->base._resv);
507
508 ttm_bo_cleanup_memtype_use(bo);
509 dma_resv_unlock(bo->base.resv);
510 return;
511 }
512
513 ttm_bo_flush_all_fences(bo);
514
515 /*
516 * Make NO_EVICT bos immediately available to
517 * shrinkers, now that they are queued for
518 * destruction.
519 */
520 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
521 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
522 ttm_bo_move_to_lru_tail(bo, NULL);
523 }
524
525 dma_resv_unlock(bo->base.resv);
526 }
527 if (bo->base.resv != &bo->base._resv)
528 dma_resv_unlock(&bo->base._resv);
529
530 error:
531 kref_get(&bo->list_kref);
532 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
533 spin_unlock(&ttm_bo_glob.lru_lock);
534
535 schedule_delayed_work(&bdev->wq,
536 ((HZ / 100) < 1) ? 1 : HZ / 100);
537 }
538
539 /**
540 * function ttm_bo_cleanup_refs
541 * If bo idle, remove from delayed- and lru lists, and unref.
542 * If not idle, do nothing.
543 *
544 * Must be called with lru_lock and reservation held, this function
545 * will drop the lru lock and optionally the reservation lock before returning.
546 *
547 * @interruptible Any sleeps should occur interruptibly.
548 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
549 * @unlock_resv Unlock the reservation lock as well.
550 */
551
ttm_bo_cleanup_refs(struct ttm_buffer_object * bo,bool interruptible,bool no_wait_gpu,bool unlock_resv)552 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
553 bool interruptible, bool no_wait_gpu,
554 bool unlock_resv)
555 {
556 struct dma_resv *resv;
557 int ret;
558
559 if (unlikely(list_empty(&bo->ddestroy)))
560 resv = bo->base.resv;
561 else
562 resv = &bo->base._resv;
563
564 if (dma_resv_test_signaled_rcu(resv, true))
565 ret = 0;
566 else
567 ret = -EBUSY;
568
569 if (ret && !no_wait_gpu) {
570 long lret;
571
572 if (unlock_resv)
573 dma_resv_unlock(bo->base.resv);
574 spin_unlock(&ttm_bo_glob.lru_lock);
575
576 lret = dma_resv_wait_timeout_rcu(resv, true,
577 interruptible,
578 30 * HZ);
579
580 if (lret < 0)
581 return lret;
582 else if (lret == 0)
583 return -EBUSY;
584
585 spin_lock(&ttm_bo_glob.lru_lock);
586 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
587 /*
588 * We raced, and lost, someone else holds the reservation now,
589 * and is probably busy in ttm_bo_cleanup_memtype_use.
590 *
591 * Even if it's not the case, because we finished waiting any
592 * delayed destruction would succeed, so just return success
593 * here.
594 */
595 spin_unlock(&ttm_bo_glob.lru_lock);
596 return 0;
597 }
598 ret = 0;
599 }
600
601 if (ret || unlikely(list_empty(&bo->ddestroy))) {
602 if (unlock_resv)
603 dma_resv_unlock(bo->base.resv);
604 spin_unlock(&ttm_bo_glob.lru_lock);
605 return ret;
606 }
607
608 ttm_bo_del_from_lru(bo);
609 list_del_init(&bo->ddestroy);
610 kref_put(&bo->list_kref, ttm_bo_ref_bug);
611
612 spin_unlock(&ttm_bo_glob.lru_lock);
613 ttm_bo_cleanup_memtype_use(bo);
614
615 if (unlock_resv)
616 dma_resv_unlock(bo->base.resv);
617
618 return 0;
619 }
620
621 /**
622 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
623 * encountered buffers.
624 */
ttm_bo_delayed_delete(struct ttm_bo_device * bdev,bool remove_all)625 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
626 {
627 struct ttm_bo_global *glob = &ttm_bo_glob;
628 struct list_head removed;
629 bool empty;
630
631 INIT_LIST_HEAD(&removed);
632
633 spin_lock(&glob->lru_lock);
634 while (!list_empty(&bdev->ddestroy)) {
635 struct ttm_buffer_object *bo;
636
637 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
638 ddestroy);
639 kref_get(&bo->list_kref);
640 list_move_tail(&bo->ddestroy, &removed);
641
642 if (remove_all || bo->base.resv != &bo->base._resv) {
643 spin_unlock(&glob->lru_lock);
644 dma_resv_lock(bo->base.resv, NULL);
645
646 spin_lock(&glob->lru_lock);
647 ttm_bo_cleanup_refs(bo, false, !remove_all, true);
648
649 } else if (dma_resv_trylock(bo->base.resv)) {
650 ttm_bo_cleanup_refs(bo, false, !remove_all, true);
651 } else {
652 spin_unlock(&glob->lru_lock);
653 }
654
655 kref_put(&bo->list_kref, ttm_bo_release_list);
656 spin_lock(&glob->lru_lock);
657 }
658 list_splice_tail(&removed, &bdev->ddestroy);
659 empty = list_empty(&bdev->ddestroy);
660 spin_unlock(&glob->lru_lock);
661
662 return empty;
663 }
664
ttm_bo_delayed_workqueue(struct work_struct * work)665 static void ttm_bo_delayed_workqueue(struct work_struct *work)
666 {
667 struct ttm_bo_device *bdev =
668 container_of(work, struct ttm_bo_device, wq.work);
669
670 if (!ttm_bo_delayed_delete(bdev, false))
671 schedule_delayed_work(&bdev->wq,
672 ((HZ / 100) < 1) ? 1 : HZ / 100);
673 }
674
ttm_bo_release(struct kref * kref)675 static void ttm_bo_release(struct kref *kref)
676 {
677 struct ttm_buffer_object *bo =
678 container_of(kref, struct ttm_buffer_object, kref);
679 struct ttm_bo_device *bdev = bo->bdev;
680 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
681
682 if (bo->bdev->driver->release_notify)
683 bo->bdev->driver->release_notify(bo);
684
685 #ifdef __NetBSD__
686 uvm_obj_destroy(&bo->uvmobj, true);
687 #endif
688 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
689 #ifdef __NetBSD__
690 if (!ttm_bo_uses_embedded_gem_object(bo))
691 drm_vma_node_destroy(&bo->base.vma_node);
692 #endif
693 ttm_mem_io_lock(man, false);
694 ttm_mem_io_free_vm(bo);
695 ttm_mem_io_unlock(man);
696 ttm_bo_cleanup_refs_or_queue(bo);
697 kref_put(&bo->list_kref, ttm_bo_release_list);
698 }
699
ttm_bo_put(struct ttm_buffer_object * bo)700 void ttm_bo_put(struct ttm_buffer_object *bo)
701 {
702 kref_put(&bo->kref, ttm_bo_release);
703 }
704 EXPORT_SYMBOL(ttm_bo_put);
705
ttm_bo_lock_delayed_workqueue(struct ttm_bo_device * bdev)706 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
707 {
708 return cancel_delayed_work_sync(&bdev->wq);
709 }
710 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
711
ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device * bdev,int resched)712 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
713 {
714 if (resched)
715 schedule_delayed_work(&bdev->wq,
716 ((HZ / 100) < 1) ? 1 : HZ / 100);
717 }
718 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
719
ttm_bo_evict(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx)720 static int ttm_bo_evict(struct ttm_buffer_object *bo,
721 struct ttm_operation_ctx *ctx)
722 {
723 struct ttm_bo_device *bdev = bo->bdev;
724 struct ttm_mem_reg evict_mem;
725 struct ttm_placement placement;
726 int ret = 0;
727
728 dma_resv_assert_held(bo->base.resv);
729
730 placement.num_placement = 0;
731 placement.num_busy_placement = 0;
732 bdev->driver->evict_flags(bo, &placement);
733
734 if (!placement.num_placement && !placement.num_busy_placement) {
735 ret = ttm_bo_pipeline_gutting(bo);
736 if (ret)
737 return ret;
738
739 return ttm_tt_create(bo, false);
740 }
741
742 evict_mem = bo->mem;
743 evict_mem.mm_node = NULL;
744 evict_mem.bus.io_reserved_vm = false;
745 evict_mem.bus.io_reserved_count = 0;
746
747 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
748 if (ret) {
749 if (ret != -ERESTARTSYS) {
750 pr_err("Failed to find memory space for buffer 0x%p eviction\n",
751 bo);
752 ttm_bo_mem_space_debug(bo, &placement);
753 }
754 goto out;
755 }
756
757 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
758 if (unlikely(ret)) {
759 if (ret != -ERESTARTSYS)
760 pr_err("Buffer eviction failed\n");
761 ttm_bo_mem_put(bo, &evict_mem);
762 goto out;
763 }
764 bo->evicted = true;
765 out:
766 return ret;
767 }
768
ttm_bo_eviction_valuable(struct ttm_buffer_object * bo,const struct ttm_place * place)769 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
770 const struct ttm_place *place)
771 {
772 /* Don't evict this BO if it's outside of the
773 * requested placement range
774 */
775 if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
776 (place->lpfn && place->lpfn <= bo->mem.start))
777 return false;
778
779 return true;
780 }
781 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
782
783 /**
784 * Check the target bo is allowable to be evicted or swapout, including cases:
785 *
786 * a. if share same reservation object with ctx->resv, have assumption
787 * reservation objects should already be locked, so not lock again and
788 * return true directly when either the opreation allow_reserved_eviction
789 * or the target bo already is in delayed free list;
790 *
791 * b. Otherwise, trylock it.
792 */
ttm_bo_evict_swapout_allowable(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,bool * locked,bool * busy)793 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
794 struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
795 {
796 bool ret = false;
797
798 if (bo->base.resv == ctx->resv) {
799 dma_resv_assert_held(bo->base.resv);
800 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
801 || !list_empty(&bo->ddestroy))
802 ret = true;
803 *locked = false;
804 if (busy)
805 *busy = false;
806 } else {
807 ret = dma_resv_trylock(bo->base.resv);
808 *locked = ret;
809 if (busy)
810 *busy = !ret;
811 }
812
813 return ret;
814 }
815
816 /**
817 * ttm_mem_evict_wait_busy - wait for a busy BO to become available
818 *
819 * @busy_bo: BO which couldn't be locked with trylock
820 * @ctx: operation context
821 * @ticket: acquire ticket
822 *
823 * Try to lock a busy buffer object to avoid failing eviction.
824 */
ttm_mem_evict_wait_busy(struct ttm_buffer_object * busy_bo,struct ttm_operation_ctx * ctx,struct ww_acquire_ctx * ticket)825 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
826 struct ttm_operation_ctx *ctx,
827 struct ww_acquire_ctx *ticket)
828 {
829 int r;
830
831 if (!busy_bo || !ticket)
832 return -EBUSY;
833
834 if (ctx->interruptible)
835 r = dma_resv_lock_interruptible(busy_bo->base.resv,
836 ticket);
837 else
838 r = dma_resv_lock(busy_bo->base.resv, ticket);
839
840 /*
841 * TODO: It would be better to keep the BO locked until allocation is at
842 * least tried one more time, but that would mean a much larger rework
843 * of TTM.
844 */
845 if (!r)
846 dma_resv_unlock(busy_bo->base.resv);
847
848 return r == -EDEADLK ? -EBUSY : r;
849 }
850
ttm_mem_evict_first(struct ttm_bo_device * bdev,uint32_t mem_type,const struct ttm_place * place,struct ttm_operation_ctx * ctx,struct ww_acquire_ctx * ticket)851 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
852 uint32_t mem_type,
853 const struct ttm_place *place,
854 struct ttm_operation_ctx *ctx,
855 struct ww_acquire_ctx *ticket)
856 {
857 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
858 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
859 bool locked = false;
860 unsigned i;
861 int ret;
862
863 spin_lock(&ttm_bo_glob.lru_lock);
864 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
865 list_for_each_entry(bo, &man->lru[i], lru) {
866 bool busy;
867
868 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
869 &busy)) {
870 if (busy && !busy_bo && ticket !=
871 dma_resv_locking_ctx(bo->base.resv))
872 busy_bo = bo;
873 continue;
874 }
875
876 if (place && !bdev->driver->eviction_valuable(bo,
877 place)) {
878 if (locked)
879 dma_resv_unlock(bo->base.resv);
880 continue;
881 }
882 break;
883 }
884
885 /* If the inner loop terminated early, we have our candidate */
886 if (&bo->lru != &man->lru[i])
887 break;
888
889 bo = NULL;
890 }
891
892 if (!bo) {
893 if (busy_bo)
894 kref_get(&busy_bo->list_kref);
895 spin_unlock(&ttm_bo_glob.lru_lock);
896 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
897 if (busy_bo)
898 kref_put(&busy_bo->list_kref, ttm_bo_release_list);
899 return ret;
900 }
901
902 kref_get(&bo->list_kref);
903
904 if (!list_empty(&bo->ddestroy)) {
905 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
906 ctx->no_wait_gpu, locked);
907 kref_put(&bo->list_kref, ttm_bo_release_list);
908 return ret;
909 }
910
911 spin_unlock(&ttm_bo_glob.lru_lock);
912
913 ret = ttm_bo_evict(bo, ctx);
914 if (locked)
915 ttm_bo_unreserve(bo);
916
917 kref_put(&bo->list_kref, ttm_bo_release_list);
918 return ret;
919 }
920
ttm_bo_mem_put(struct ttm_buffer_object * bo,struct ttm_mem_reg * mem)921 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
922 {
923 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
924
925 if (mem->mm_node)
926 (*man->func->put_node)(man, mem);
927 }
928 EXPORT_SYMBOL(ttm_bo_mem_put);
929
930 /**
931 * Add the last move fence to the BO and reserve a new shared slot.
932 */
ttm_bo_add_move_fence(struct ttm_buffer_object * bo,struct ttm_mem_type_manager * man,struct ttm_mem_reg * mem,bool no_wait_gpu)933 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
934 struct ttm_mem_type_manager *man,
935 struct ttm_mem_reg *mem,
936 bool no_wait_gpu)
937 {
938 struct dma_fence *fence;
939 int ret;
940
941 spin_lock(&man->move_lock);
942 fence = dma_fence_get(man->move);
943 spin_unlock(&man->move_lock);
944
945 if (!fence)
946 return 0;
947
948 if (no_wait_gpu)
949 return -EBUSY;
950
951 dma_resv_add_shared_fence(bo->base.resv, fence);
952
953 ret = dma_resv_reserve_shared(bo->base.resv, 1);
954 if (unlikely(ret)) {
955 dma_fence_put(fence);
956 return ret;
957 }
958
959 dma_fence_put(bo->moving);
960 bo->moving = fence;
961 return 0;
962 }
963
964 /**
965 * Repeatedly evict memory from the LRU for @mem_type until we create enough
966 * space, or we've evicted everything and there isn't enough space.
967 */
ttm_bo_mem_force_space(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_mem_reg * mem,struct ttm_operation_ctx * ctx)968 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
969 const struct ttm_place *place,
970 struct ttm_mem_reg *mem,
971 struct ttm_operation_ctx *ctx)
972 {
973 struct ttm_bo_device *bdev = bo->bdev;
974 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
975 struct ww_acquire_ctx *ticket;
976 int ret;
977
978 ticket = dma_resv_locking_ctx(bo->base.resv);
979 do {
980 ret = (*man->func->get_node)(man, bo, place, mem);
981 if (unlikely(ret != 0))
982 return ret;
983 if (mem->mm_node)
984 break;
985 ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx,
986 ticket);
987 if (unlikely(ret != 0))
988 return ret;
989 } while (1);
990
991 return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
992 }
993
ttm_bo_select_caching(struct ttm_mem_type_manager * man,uint32_t cur_placement,uint32_t proposed_placement)994 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
995 uint32_t cur_placement,
996 uint32_t proposed_placement)
997 {
998 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
999 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
1000
1001 /**
1002 * Keep current caching if possible.
1003 */
1004
1005 if ((cur_placement & caching) != 0)
1006 result |= (cur_placement & caching);
1007 else if ((man->default_caching & caching) != 0)
1008 result |= man->default_caching;
1009 else if ((TTM_PL_FLAG_CACHED & caching) != 0)
1010 result |= TTM_PL_FLAG_CACHED;
1011 else if ((TTM_PL_FLAG_WC & caching) != 0)
1012 result |= TTM_PL_FLAG_WC;
1013 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
1014 result |= TTM_PL_FLAG_UNCACHED;
1015
1016 return result;
1017 }
1018
ttm_bo_mt_compatible(struct ttm_mem_type_manager * man,uint32_t mem_type,const struct ttm_place * place,uint32_t * masked_placement)1019 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
1020 uint32_t mem_type,
1021 const struct ttm_place *place,
1022 uint32_t *masked_placement)
1023 {
1024 uint32_t cur_flags = ttm_bo_type_flags(mem_type);
1025
1026 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
1027 return false;
1028
1029 if ((place->flags & man->available_caching) == 0)
1030 return false;
1031
1032 cur_flags |= (place->flags & man->available_caching);
1033
1034 *masked_placement = cur_flags;
1035 return true;
1036 }
1037
1038 /**
1039 * ttm_bo_mem_placement - check if placement is compatible
1040 * @bo: BO to find memory for
1041 * @place: where to search
1042 * @mem: the memory object to fill in
1043 * @ctx: operation context
1044 *
1045 * Check if placement is compatible and fill in mem structure.
1046 * Returns -EBUSY if placement won't work or negative error code.
1047 * 0 when placement can be used.
1048 */
ttm_bo_mem_placement(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_mem_reg * mem,struct ttm_operation_ctx * ctx)1049 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
1050 const struct ttm_place *place,
1051 struct ttm_mem_reg *mem,
1052 struct ttm_operation_ctx *ctx)
1053 {
1054 struct ttm_bo_device *bdev = bo->bdev;
1055 uint32_t mem_type = TTM_PL_SYSTEM;
1056 struct ttm_mem_type_manager *man;
1057 uint32_t cur_flags = 0;
1058 int ret;
1059
1060 ret = ttm_mem_type_from_place(place, &mem_type);
1061 if (ret)
1062 return ret;
1063
1064 man = &bdev->man[mem_type];
1065 if (!man->has_type || !man->use_type)
1066 return -EBUSY;
1067
1068 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1069 return -EBUSY;
1070
1071 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);
1072 /*
1073 * Use the access and other non-mapping-related flag bits from
1074 * the memory placement flags to the current flags
1075 */
1076 ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE);
1077
1078 mem->mem_type = mem_type;
1079 mem->placement = cur_flags;
1080
1081 spin_lock(&ttm_bo_glob.lru_lock);
1082 ttm_bo_del_from_lru(bo);
1083 ttm_bo_add_mem_to_lru(bo, mem);
1084 spin_unlock(&ttm_bo_glob.lru_lock);
1085
1086 return 0;
1087 }
1088
1089 /**
1090 * Creates space for memory region @mem according to its type.
1091 *
1092 * This function first searches for free space in compatible memory types in
1093 * the priority order defined by the driver. If free space isn't found, then
1094 * ttm_bo_mem_force_space is attempted in priority order to evict and find
1095 * space.
1096 */
ttm_bo_mem_space(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_mem_reg * mem,struct ttm_operation_ctx * ctx)1097 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
1098 struct ttm_placement *placement,
1099 struct ttm_mem_reg *mem,
1100 struct ttm_operation_ctx *ctx)
1101 {
1102 struct ttm_bo_device *bdev = bo->bdev;
1103 bool type_found = false;
1104 int i, ret;
1105
1106 ret = dma_resv_reserve_shared(bo->base.resv, 1);
1107 if (unlikely(ret))
1108 return ret;
1109
1110 mem->mm_node = NULL;
1111 for (i = 0; i < placement->num_placement; ++i) {
1112 const struct ttm_place *place = &placement->placement[i];
1113 struct ttm_mem_type_manager *man;
1114
1115 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1116 if (ret == -EBUSY)
1117 continue;
1118 if (ret)
1119 goto error;
1120
1121 type_found = true;
1122 mem->mm_node = NULL;
1123 if (mem->mem_type == TTM_PL_SYSTEM)
1124 return 0;
1125
1126 man = &bdev->man[mem->mem_type];
1127 ret = (*man->func->get_node)(man, bo, place, mem);
1128 if (unlikely(ret))
1129 goto error;
1130
1131 if (!mem->mm_node)
1132 continue;
1133
1134 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
1135 if (unlikely(ret)) {
1136 (*man->func->put_node)(man, mem);
1137 if (ret == -EBUSY)
1138 continue;
1139
1140 goto error;
1141 }
1142 return 0;
1143 }
1144
1145 for (i = 0; i < placement->num_busy_placement; ++i) {
1146 const struct ttm_place *place = &placement->busy_placement[i];
1147
1148 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1149 if (ret == -EBUSY)
1150 continue;
1151 if (ret)
1152 goto error;
1153
1154 type_found = true;
1155 mem->mm_node = NULL;
1156 if (mem->mem_type == TTM_PL_SYSTEM)
1157 return 0;
1158
1159 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
1160 if (ret == 0 && mem->mm_node)
1161 return 0;
1162
1163 if (ret && ret != -EBUSY)
1164 goto error;
1165 }
1166
1167 ret = -ENOMEM;
1168 if (!type_found) {
1169 pr_err(TTM_PFX "No compatible memory type found\n");
1170 ret = -EINVAL;
1171 }
1172
1173 error:
1174 if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
1175 spin_lock(&ttm_bo_glob.lru_lock);
1176 ttm_bo_move_to_lru_tail(bo, NULL);
1177 spin_unlock(&ttm_bo_glob.lru_lock);
1178 }
1179
1180 return ret;
1181 }
1182 EXPORT_SYMBOL(ttm_bo_mem_space);
1183
ttm_bo_move_buffer(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_operation_ctx * ctx)1184 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1185 struct ttm_placement *placement,
1186 struct ttm_operation_ctx *ctx)
1187 {
1188 int ret = 0;
1189 struct ttm_mem_reg mem;
1190
1191 dma_resv_assert_held(bo->base.resv);
1192
1193 memset(&mem, 0, sizeof(mem));
1194 mem.num_pages = bo->num_pages;
1195 mem.size = mem.num_pages << PAGE_SHIFT;
1196 mem.page_alignment = bo->mem.page_alignment;
1197 mem.bus.is_iomem = false;
1198 mem.bus.io_reserved_vm = false;
1199 mem.bus.io_reserved_count = 0;
1200 /*
1201 * Determine where to move the buffer.
1202 */
1203 ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1204 if (ret)
1205 goto out_unlock;
1206 ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1207 out_unlock:
1208 if (ret && mem.mm_node)
1209 ttm_bo_mem_put(bo, &mem);
1210 return ret;
1211 }
1212
ttm_bo_places_compat(const struct ttm_place * places,unsigned num_placement,struct ttm_mem_reg * mem,uint32_t * new_flags)1213 static bool ttm_bo_places_compat(const struct ttm_place *places,
1214 unsigned num_placement,
1215 struct ttm_mem_reg *mem,
1216 uint32_t *new_flags)
1217 {
1218 unsigned i;
1219
1220 for (i = 0; i < num_placement; i++) {
1221 const struct ttm_place *heap = &places[i];
1222
1223 if (mem->mm_node && (mem->start < heap->fpfn ||
1224 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1225 continue;
1226
1227 *new_flags = heap->flags;
1228 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1229 (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1230 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1231 (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1232 return true;
1233 }
1234 return false;
1235 }
1236
ttm_bo_mem_compat(struct ttm_placement * placement,struct ttm_mem_reg * mem,uint32_t * new_flags)1237 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1238 struct ttm_mem_reg *mem,
1239 uint32_t *new_flags)
1240 {
1241 if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1242 mem, new_flags))
1243 return true;
1244
1245 if ((placement->busy_placement != placement->placement ||
1246 placement->num_busy_placement > placement->num_placement) &&
1247 ttm_bo_places_compat(placement->busy_placement,
1248 placement->num_busy_placement,
1249 mem, new_flags))
1250 return true;
1251
1252 return false;
1253 }
1254 EXPORT_SYMBOL(ttm_bo_mem_compat);
1255
ttm_bo_validate(struct ttm_buffer_object * bo,struct ttm_placement * placement,struct ttm_operation_ctx * ctx)1256 int ttm_bo_validate(struct ttm_buffer_object *bo,
1257 struct ttm_placement *placement,
1258 struct ttm_operation_ctx *ctx)
1259 {
1260 int ret;
1261 uint32_t new_flags;
1262
1263 dma_resv_assert_held(bo->base.resv);
1264 /*
1265 * Check whether we need to move buffer.
1266 */
1267 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1268 ret = ttm_bo_move_buffer(bo, placement, ctx);
1269 if (ret)
1270 return ret;
1271 } else {
1272 /*
1273 * Use the access and other non-mapping-related flag bits from
1274 * the compatible memory placement flags to the active flags
1275 */
1276 ttm_flag_masked(&bo->mem.placement, new_flags,
1277 ~TTM_PL_MASK_MEMTYPE);
1278 }
1279 /*
1280 * We might need to add a TTM.
1281 */
1282 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1283 ret = ttm_tt_create(bo, true);
1284 if (ret)
1285 return ret;
1286 }
1287 return 0;
1288 }
1289 EXPORT_SYMBOL(ttm_bo_validate);
1290
ttm_bo_init_reserved(struct ttm_bo_device * bdev,struct ttm_buffer_object * bo,unsigned long size,enum ttm_bo_type type,struct ttm_placement * placement,uint32_t page_alignment,struct ttm_operation_ctx * ctx,size_t acc_size,struct sg_table * sg,struct dma_resv * resv,void (* destroy)(struct ttm_buffer_object *))1291 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1292 struct ttm_buffer_object *bo,
1293 unsigned long size,
1294 enum ttm_bo_type type,
1295 struct ttm_placement *placement,
1296 uint32_t page_alignment,
1297 struct ttm_operation_ctx *ctx,
1298 size_t acc_size,
1299 struct sg_table *sg,
1300 struct dma_resv *resv,
1301 void (*destroy) (struct ttm_buffer_object *))
1302 {
1303 struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1304 int ret = 0;
1305 unsigned long num_pages;
1306 bool locked;
1307
1308 if (sg && !drm_prime_sg_importable(bdev->dmat, sg)) {
1309 pr_err("DRM prime buffer violates DMA constraints\n");
1310 return -EIO;
1311 }
1312
1313 ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1314 if (ret) {
1315 pr_err("Out of kernel memory\n");
1316 if (destroy)
1317 (*destroy)(bo);
1318 else
1319 kfree(bo);
1320 return -ENOMEM;
1321 }
1322
1323 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1324 if (num_pages == 0) {
1325 pr_err("Illegal buffer object size\n");
1326 if (destroy)
1327 (*destroy)(bo);
1328 else
1329 kfree(bo);
1330 ttm_mem_global_free(mem_glob, acc_size);
1331 return -EINVAL;
1332 }
1333 bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1334
1335 kref_init(&bo->kref);
1336 kref_init(&bo->list_kref);
1337 INIT_LIST_HEAD(&bo->lru);
1338 INIT_LIST_HEAD(&bo->ddestroy);
1339 INIT_LIST_HEAD(&bo->swap);
1340 INIT_LIST_HEAD(&bo->io_reserve_lru);
1341 bo->bdev = bdev;
1342 bo->type = type;
1343 bo->num_pages = num_pages;
1344 bo->mem.size = num_pages << PAGE_SHIFT;
1345 bo->mem.mem_type = TTM_PL_SYSTEM;
1346 bo->mem.num_pages = bo->num_pages;
1347 bo->mem.mm_node = NULL;
1348 bo->mem.page_alignment = page_alignment;
1349 bo->mem.bus.io_reserved_vm = false;
1350 bo->mem.bus.io_reserved_count = 0;
1351 bo->moving = NULL;
1352 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1353 bo->acc_size = acc_size;
1354 bo->sg = sg;
1355 if (resv) {
1356 bo->base.resv = resv;
1357 dma_resv_assert_held(bo->base.resv);
1358 } else {
1359 bo->base.resv = &bo->base._resv;
1360 }
1361 if (!ttm_bo_uses_embedded_gem_object(bo)) {
1362 /*
1363 * bo.gem is not initialized, so we have to setup the
1364 * struct elements we want use regardless.
1365 */
1366 dma_resv_init(&bo->base._resv);
1367 #ifdef __NetBSD__
1368 drm_vma_node_init(&bo->base.vma_node);
1369 #else
1370 drm_vma_node_reset(&bo->base.vma_node);
1371 #endif
1372 }
1373 #ifdef __NetBSD__
1374 uvm_obj_init(&bo->uvmobj, bdev->driver->ttm_uvm_ops, true, 1);
1375 #endif
1376 atomic_inc(&ttm_bo_glob.bo_count);
1377
1378 /*
1379 * For ttm_bo_type_device buffers, allocate
1380 * address space from the device.
1381 */
1382 if (bo->type == ttm_bo_type_device ||
1383 bo->type == ttm_bo_type_sg)
1384 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1385 bo->mem.num_pages);
1386
1387 /* passed reservation objects should already be locked,
1388 * since otherwise lockdep will be angered in radeon.
1389 */
1390 if (!resv) {
1391 locked = dma_resv_trylock(bo->base.resv);
1392 WARN_ON(!locked);
1393 }
1394
1395 if (likely(!ret))
1396 ret = ttm_bo_validate(bo, placement, ctx);
1397
1398 if (unlikely(ret)) {
1399 if (!resv)
1400 ttm_bo_unreserve(bo);
1401
1402 ttm_bo_put(bo);
1403 return ret;
1404 }
1405
1406 spin_lock(&ttm_bo_glob.lru_lock);
1407 ttm_bo_move_to_lru_tail(bo, NULL);
1408 spin_unlock(&ttm_bo_glob.lru_lock);
1409
1410 return ret;
1411 }
1412 EXPORT_SYMBOL(ttm_bo_init_reserved);
1413
ttm_bo_init(struct ttm_bo_device * bdev,struct ttm_buffer_object * bo,unsigned long size,enum ttm_bo_type type,struct ttm_placement * placement,uint32_t page_alignment,bool interruptible,size_t acc_size,struct sg_table * sg,struct dma_resv * resv,void (* destroy)(struct ttm_buffer_object *))1414 int ttm_bo_init(struct ttm_bo_device *bdev,
1415 struct ttm_buffer_object *bo,
1416 unsigned long size,
1417 enum ttm_bo_type type,
1418 struct ttm_placement *placement,
1419 uint32_t page_alignment,
1420 bool interruptible,
1421 size_t acc_size,
1422 struct sg_table *sg,
1423 struct dma_resv *resv,
1424 void (*destroy) (struct ttm_buffer_object *))
1425 {
1426 struct ttm_operation_ctx ctx = { interruptible, false };
1427 int ret;
1428
1429 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1430 page_alignment, &ctx, acc_size,
1431 sg, resv, destroy);
1432 if (ret)
1433 return ret;
1434
1435 if (!resv)
1436 ttm_bo_unreserve(bo);
1437
1438 return 0;
1439 }
1440 EXPORT_SYMBOL(ttm_bo_init);
1441
ttm_bo_acc_size(struct ttm_bo_device * bdev,unsigned long bo_size,unsigned struct_size)1442 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1443 unsigned long bo_size,
1444 unsigned struct_size)
1445 {
1446 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1447 size_t size = 0;
1448
1449 size += ttm_round_pot(struct_size);
1450 size += ttm_round_pot(npages * sizeof(void *));
1451 size += ttm_round_pot(sizeof(struct ttm_tt));
1452 return size;
1453 }
1454 EXPORT_SYMBOL(ttm_bo_acc_size);
1455
ttm_bo_dma_acc_size(struct ttm_bo_device * bdev,unsigned long bo_size,unsigned struct_size)1456 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1457 unsigned long bo_size,
1458 unsigned struct_size)
1459 {
1460 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1461 size_t size = 0;
1462
1463 size += ttm_round_pot(struct_size);
1464 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1465 size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1466 return size;
1467 }
1468 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1469
ttm_bo_create(struct ttm_bo_device * bdev,unsigned long size,enum ttm_bo_type type,struct ttm_placement * placement,uint32_t page_alignment,bool interruptible,struct ttm_buffer_object ** p_bo)1470 int ttm_bo_create(struct ttm_bo_device *bdev,
1471 unsigned long size,
1472 enum ttm_bo_type type,
1473 struct ttm_placement *placement,
1474 uint32_t page_alignment,
1475 bool interruptible,
1476 struct ttm_buffer_object **p_bo)
1477 {
1478 struct ttm_buffer_object *bo;
1479 size_t acc_size;
1480 int ret;
1481
1482 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1483 if (unlikely(bo == NULL))
1484 return -ENOMEM;
1485
1486 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1487 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1488 interruptible, acc_size,
1489 NULL, NULL, NULL);
1490 if (likely(ret == 0))
1491 *p_bo = bo;
1492
1493 return ret;
1494 }
1495 EXPORT_SYMBOL(ttm_bo_create);
1496
ttm_bo_force_list_clean(struct ttm_bo_device * bdev,unsigned mem_type)1497 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1498 unsigned mem_type)
1499 {
1500 struct ttm_operation_ctx ctx = {
1501 .interruptible = false,
1502 .no_wait_gpu = false,
1503 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1504 };
1505 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1506 struct ttm_bo_global *glob = &ttm_bo_glob;
1507 struct dma_fence *fence;
1508 int ret;
1509 unsigned i;
1510
1511 /*
1512 * Can't use standard list traversal since we're unlocking.
1513 */
1514
1515 spin_lock(&glob->lru_lock);
1516 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1517 while (!list_empty(&man->lru[i])) {
1518 spin_unlock(&glob->lru_lock);
1519 ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx,
1520 NULL);
1521 if (ret)
1522 return ret;
1523 spin_lock(&glob->lru_lock);
1524 }
1525 }
1526 spin_unlock(&glob->lru_lock);
1527
1528 spin_lock(&man->move_lock);
1529 fence = dma_fence_get(man->move);
1530 spin_unlock(&man->move_lock);
1531
1532 if (fence) {
1533 ret = dma_fence_wait(fence, false);
1534 dma_fence_put(fence);
1535 if (ret)
1536 return ret;
1537 }
1538
1539 return 0;
1540 }
1541
ttm_bo_clean_mm(struct ttm_bo_device * bdev,unsigned mem_type)1542 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1543 {
1544 struct ttm_mem_type_manager *man;
1545 int ret = -EINVAL;
1546
1547 if (mem_type >= TTM_NUM_MEM_TYPES) {
1548 pr_err("Illegal memory type %d\n", mem_type);
1549 return ret;
1550 }
1551 man = &bdev->man[mem_type];
1552
1553 if (!man->has_type) {
1554 pr_err("Trying to take down uninitialized memory manager type %u\n",
1555 mem_type);
1556 return ret;
1557 }
1558
1559 man->use_type = false;
1560 man->has_type = false;
1561
1562 ret = 0;
1563 if (mem_type > 0) {
1564 ret = ttm_bo_force_list_clean(bdev, mem_type);
1565 if (ret) {
1566 pr_err("Cleanup eviction failed\n");
1567 return ret;
1568 }
1569
1570 ret = (*man->func->takedown)(man);
1571 }
1572
1573 dma_fence_put(man->move);
1574 man->move = NULL;
1575
1576 return ret;
1577 }
1578 EXPORT_SYMBOL(ttm_bo_clean_mm);
1579
ttm_bo_evict_mm(struct ttm_bo_device * bdev,unsigned mem_type)1580 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1581 {
1582 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1583
1584 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1585 pr_err("Illegal memory manager memory type %u\n", mem_type);
1586 return -EINVAL;
1587 }
1588
1589 if (!man->has_type) {
1590 pr_err("Memory type %u has not been initialized\n", mem_type);
1591 return 0;
1592 }
1593
1594 return ttm_bo_force_list_clean(bdev, mem_type);
1595 }
1596 EXPORT_SYMBOL(ttm_bo_evict_mm);
1597
ttm_bo_init_mm(struct ttm_bo_device * bdev,unsigned type,unsigned long p_size)1598 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1599 unsigned long p_size)
1600 {
1601 int ret;
1602 struct ttm_mem_type_manager *man;
1603 unsigned i;
1604
1605 BUG_ON(type >= TTM_NUM_MEM_TYPES);
1606 man = &bdev->man[type];
1607 BUG_ON(man->has_type);
1608 man->io_reserve_fastpath = true;
1609 man->use_io_reserve_lru = false;
1610 mutex_init(&man->io_reserve_mutex);
1611 spin_lock_init(&man->move_lock);
1612 INIT_LIST_HEAD(&man->io_reserve_lru);
1613
1614 ret = bdev->driver->init_mem_type(bdev, type, man);
1615 if (ret)
1616 return ret;
1617 man->bdev = bdev;
1618
1619 if (type != TTM_PL_SYSTEM) {
1620 ret = (*man->func->init)(man, p_size);
1621 if (ret)
1622 return ret;
1623 }
1624 man->has_type = true;
1625 man->use_type = true;
1626 man->size = p_size;
1627
1628 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1629 INIT_LIST_HEAD(&man->lru[i]);
1630 man->move = NULL;
1631
1632 return 0;
1633 }
1634 EXPORT_SYMBOL(ttm_bo_init_mm);
1635
1636 #ifndef __NetBSD__
ttm_bo_global_kobj_release(struct kobject * kobj)1637 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1638 {
1639 struct ttm_bo_global *glob =
1640 container_of(kobj, struct ttm_bo_global, kobj);
1641
1642 __free_page(glob->dummy_read_page);
1643 }
1644 #endif
1645
ttm_bo_global_release(void)1646 static void ttm_bo_global_release(void)
1647 {
1648 struct ttm_bo_global *glob = &ttm_bo_glob;
1649
1650 mutex_lock(&ttm_global_mutex);
1651 if (--ttm_bo_glob_use_count > 0)
1652 goto out;
1653
1654 #ifndef __NetBSD__
1655 kobject_del(&glob->kobj);
1656 kobject_put(&glob->kobj);
1657 #endif
1658 ttm_mem_global_release(&ttm_mem_glob);
1659 memset(glob, 0, sizeof(*glob));
1660 #ifdef __NetBSD__
1661 BUG_ON(glob->dummy_read_page != NULL);
1662 spin_lock_destroy(&glob->lru_lock);
1663 mutex_unlock(&ttm_global_mutex);
1664 mutex_destroy(&ttm_global_mutex);
1665 return;
1666 #endif
1667 out:
1668 mutex_unlock(&ttm_global_mutex);
1669 }
1670
ttm_bo_global_init(void)1671 static int ttm_bo_global_init(void)
1672 {
1673 struct ttm_bo_global *glob = &ttm_bo_glob;
1674 int ret = 0;
1675 unsigned i;
1676
1677 mutex_init(&ttm_global_mutex);
1678 mutex_lock(&ttm_global_mutex);
1679 if (++ttm_bo_glob_use_count > 1)
1680 goto out;
1681
1682 ret = ttm_mem_global_init(&ttm_mem_glob);
1683 if (ret)
1684 goto out;
1685
1686 spin_lock_init(&glob->lru_lock);
1687 #ifdef __NetBSD__
1688 /* Only used by agp back end, will fix there. */
1689 /* XXX Fix agp back end to DTRT. */
1690 glob->dummy_read_page = NULL;
1691 #else
1692 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1693
1694 if (unlikely(glob->dummy_read_page == NULL)) {
1695 ret = -ENOMEM;
1696 goto out;
1697 }
1698 #endif
1699
1700 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1701 INIT_LIST_HEAD(&glob->swap_lru[i]);
1702 INIT_LIST_HEAD(&glob->device_list);
1703 atomic_set(&glob->bo_count, 0);
1704
1705 #ifdef __NetBSD__
1706 ret = 0;
1707 #else
1708 ret = kobject_init_and_add(
1709 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1710 if (unlikely(ret != 0))
1711 kobject_put(&glob->kobj);
1712 #endif
1713 out:
1714 mutex_unlock(&ttm_global_mutex);
1715 return ret;
1716 }
1717
ttm_bo_device_release(struct ttm_bo_device * bdev)1718 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1719 {
1720 struct ttm_bo_global *glob = &ttm_bo_glob;
1721 int ret = 0;
1722 unsigned i = TTM_NUM_MEM_TYPES;
1723 struct ttm_mem_type_manager *man;
1724
1725 while (i--) {
1726 man = &bdev->man[i];
1727 if (man->has_type) {
1728 man->use_type = false;
1729 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1730 ret = -EBUSY;
1731 pr_err("DRM memory manager type %d is not clean\n",
1732 i);
1733 }
1734 man->has_type = false;
1735 }
1736 }
1737
1738 mutex_lock(&ttm_global_mutex);
1739 list_del(&bdev->device_list);
1740 mutex_unlock(&ttm_global_mutex);
1741
1742 cancel_delayed_work_sync(&bdev->wq);
1743
1744 if (ttm_bo_delayed_delete(bdev, true))
1745 pr_debug("Delayed destroy list was clean\n");
1746
1747 spin_lock(&glob->lru_lock);
1748 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1749 if (list_empty(&bdev->man[0].lru[0]))
1750 pr_debug("Swap list %d was clean\n", i);
1751 spin_unlock(&glob->lru_lock);
1752
1753 if (!ret)
1754 ttm_bo_global_release();
1755
1756 return ret;
1757 }
1758 EXPORT_SYMBOL(ttm_bo_device_release);
1759
ttm_bo_device_init(struct ttm_bo_device * bdev,struct ttm_bo_driver * driver,bus_space_tag_t memt,bus_dma_tag_t dmat,struct drm_vma_offset_manager * vma_manager,bool need_dma32)1760 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1761 struct ttm_bo_driver *driver,
1762 #ifdef __NetBSD__
1763 bus_space_tag_t memt,
1764 bus_dma_tag_t dmat,
1765 #else
1766 struct address_space *mapping,
1767 #endif
1768 struct drm_vma_offset_manager *vma_manager,
1769 bool need_dma32)
1770 {
1771 struct ttm_bo_global *glob = &ttm_bo_glob;
1772 int ret;
1773
1774 if (WARN_ON(vma_manager == NULL))
1775 return -EINVAL;
1776
1777 ret = ttm_bo_global_init();
1778 if (ret)
1779 return ret;
1780
1781 bdev->driver = driver;
1782
1783 memset(bdev->man, 0, sizeof(bdev->man));
1784
1785 /*
1786 * Initialize the system memory buffer type.
1787 * Other types need to be driver / IOCTL initialized.
1788 */
1789 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1790 if (unlikely(ret != 0))
1791 goto out_no_sys;
1792
1793 bdev->vma_manager = vma_manager;
1794 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1795 INIT_LIST_HEAD(&bdev->ddestroy);
1796 #ifdef __NetBSD__
1797 bdev->memt = memt;
1798 bdev->dmat = dmat;
1799 #else
1800 bdev->dev_mapping = mapping;
1801 #endif
1802 bdev->need_dma32 = need_dma32;
1803 mutex_lock(&ttm_global_mutex);
1804 list_add_tail(&bdev->device_list, &glob->device_list);
1805 mutex_unlock(&ttm_global_mutex);
1806
1807 return 0;
1808 out_no_sys:
1809 ttm_bo_global_release();
1810 return ret;
1811 }
1812 EXPORT_SYMBOL(ttm_bo_device_init);
1813
1814 /*
1815 * buffer object vm functions.
1816 */
1817
ttm_mem_reg_is_pci(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem)1818 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1819 {
1820 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1821
1822 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1823 if (mem->mem_type == TTM_PL_SYSTEM)
1824 return false;
1825
1826 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1827 return false;
1828
1829 if (mem->placement & TTM_PL_FLAG_CACHED)
1830 return false;
1831 }
1832 return true;
1833 }
1834
ttm_bo_unmap_virtual_locked(struct ttm_buffer_object * bo)1835 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1836 {
1837 #ifdef __NetBSD__
1838 if (bo->mem.bus.is_iomem) {
1839 paddr_t start, end, pa;
1840
1841 KASSERTMSG((bo->mem.bus.base & (PAGE_SIZE - 1)) == 0,
1842 "bo bus base addr not page-aligned: %" PRIx64 "",
1843 (uint64_t)bo->mem.bus.base);
1844 KASSERTMSG((bo->mem.bus.offset & (PAGE_SIZE - 1)) == 0,
1845 "bo bus offset not page-aligned: %lx",
1846 bo->mem.bus.offset);
1847 start = bo->mem.bus.base + bo->mem.bus.offset;
1848 KASSERT((bo->mem.bus.size & (PAGE_SIZE - 1)) == 0);
1849 end = start + bo->mem.bus.size;
1850
1851 for (pa = start; pa < end; pa += PAGE_SIZE)
1852 pmap_pv_protect(pa, VM_PROT_NONE);
1853 } else if (bo->ttm != NULL) {
1854 unsigned i;
1855
1856 rw_enter(bo->uvmobj.vmobjlock, RW_WRITER);
1857 for (i = 0; i < bo->ttm->num_pages; i++)
1858 pmap_page_protect(&bo->ttm->pages[i]->p_vmp,
1859 VM_PROT_NONE);
1860 rw_exit(bo->uvmobj.vmobjlock);
1861 }
1862 #else
1863 struct ttm_bo_device *bdev = bo->bdev;
1864
1865 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1866 #endif
1867 ttm_mem_io_free_vm(bo);
1868 }
1869
ttm_bo_unmap_virtual(struct ttm_buffer_object * bo)1870 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1871 {
1872 struct ttm_bo_device *bdev = bo->bdev;
1873 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1874
1875 ttm_mem_io_lock(man, false);
1876 ttm_bo_unmap_virtual_locked(bo);
1877 ttm_mem_io_unlock(man);
1878 }
1879
1880
1881 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1882
ttm_bo_wait(struct ttm_buffer_object * bo,bool interruptible,bool no_wait)1883 int ttm_bo_wait(struct ttm_buffer_object *bo,
1884 bool interruptible, bool no_wait)
1885 {
1886 long timeout = 15 * HZ;
1887
1888 if (no_wait) {
1889 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1890 return 0;
1891 else
1892 return -EBUSY;
1893 }
1894
1895 timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1896 interruptible, timeout);
1897 if (timeout < 0)
1898 return timeout;
1899
1900 if (timeout == 0)
1901 return -EBUSY;
1902
1903 dma_resv_add_excl_fence(bo->base.resv, NULL);
1904 return 0;
1905 }
1906 EXPORT_SYMBOL(ttm_bo_wait);
1907
1908 /**
1909 * A buffer object shrink method that tries to swap out the first
1910 * buffer object on the bo_global::swap_lru list.
1911 */
ttm_bo_swapout(struct ttm_bo_global * glob,struct ttm_operation_ctx * ctx)1912 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1913 {
1914 struct ttm_buffer_object *bo;
1915 int ret = -EBUSY;
1916 bool locked;
1917 unsigned i;
1918
1919 spin_lock(&glob->lru_lock);
1920 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1921 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1922 if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1923 NULL)) {
1924 ret = 0;
1925 break;
1926 }
1927 }
1928 if (!ret)
1929 break;
1930 }
1931
1932 if (ret) {
1933 spin_unlock(&glob->lru_lock);
1934 return ret;
1935 }
1936
1937 kref_get(&bo->list_kref);
1938
1939 if (!list_empty(&bo->ddestroy)) {
1940 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1941 kref_put(&bo->list_kref, ttm_bo_release_list);
1942 return ret;
1943 }
1944
1945 ttm_bo_del_from_lru(bo);
1946 spin_unlock(&glob->lru_lock);
1947
1948 /**
1949 * Move to system cached
1950 */
1951
1952 if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1953 bo->ttm->caching_state != tt_cached) {
1954 struct ttm_operation_ctx ctx = { false, false };
1955 struct ttm_mem_reg evict_mem;
1956
1957 evict_mem = bo->mem;
1958 evict_mem.mm_node = NULL;
1959 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1960 evict_mem.mem_type = TTM_PL_SYSTEM;
1961
1962 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1963 if (unlikely(ret != 0))
1964 goto out;
1965 }
1966
1967 /**
1968 * Make sure BO is idle.
1969 */
1970
1971 ret = ttm_bo_wait(bo, false, false);
1972 if (unlikely(ret != 0))
1973 goto out;
1974
1975 ttm_bo_unmap_virtual(bo);
1976
1977 /**
1978 * Swap out. Buffer will be swapped in again as soon as
1979 * anyone tries to access a ttm page.
1980 */
1981
1982 if (bo->bdev->driver->swap_notify)
1983 bo->bdev->driver->swap_notify(bo);
1984
1985 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1986 out:
1987
1988 /**
1989 *
1990 * Unreserve without putting on LRU to avoid swapping out an
1991 * already swapped buffer.
1992 */
1993 if (locked)
1994 dma_resv_unlock(bo->base.resv);
1995 kref_put(&bo->list_kref, ttm_bo_release_list);
1996 return ret;
1997 }
1998 EXPORT_SYMBOL(ttm_bo_swapout);
1999
ttm_bo_swapout_all(struct ttm_bo_device * bdev)2000 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
2001 {
2002 struct ttm_operation_ctx ctx = {
2003 .interruptible = false,
2004 .no_wait_gpu = false
2005 };
2006
2007 while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0);
2008 }
2009 EXPORT_SYMBOL(ttm_bo_swapout_all);
2010