1 /* $NetBSD: i915_gem.c,v 1.75 2021/12/19 12:32:15 riastradh Exp $ */
2
3 /*
4 * Copyright © 2008-2015 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 * IN THE SOFTWARE.
24 *
25 * Authors:
26 * Eric Anholt <eric@anholt.net>
27 *
28 */
29
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.75 2021/12/19 12:32:15 riastradh Exp $");
32
33 #ifdef __NetBSD__
34 #include <drm/bus_dma_hacks.h>
35 #endif
36
37 #include <drm/drm_vma_manager.h>
38 #include <drm/i915_drm.h>
39 #include <linux/dma-fence-array.h>
40 #include <linux/kthread.h>
41 #include <linux/dma-resv.h>
42 #include <linux/shmem_fs.h>
43 #include <linux/slab.h>
44 #include <linux/stop_machine.h>
45 #include <linux/swap.h>
46 #include <linux/pci.h>
47 #include <linux/dma-buf.h>
48 #include <linux/mman.h>
49 #include <linux/uaccess.h>
50
51 #include "display/intel_display.h"
52 #include "display/intel_frontbuffer.h"
53
54 #include "gem/i915_gem_clflush.h"
55 #include "gem/i915_gem_context.h"
56 #include "gem/i915_gem_ioctls.h"
57 #include "gem/i915_gem_mman.h"
58 #include "gem/i915_gem_region.h"
59 #include "gt/intel_engine_user.h"
60 #include "gt/intel_gt.h"
61 #include "gt/intel_gt_pm.h"
62 #include "gt/intel_workarounds.h"
63
64 #include "i915_drv.h"
65 #include "i915_trace.h"
66 #include "i915_vgpu.h"
67
68 #include "intel_pm.h"
69
70 #include <linux/nbsd-namespace.h>
71
72 static int
insert_mappable_node(struct i915_ggtt * ggtt,struct drm_mm_node * node,u32 size)73 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
74 {
75 int err;
76
77 err = mutex_lock_interruptible(&ggtt->vm.mutex);
78 if (err)
79 return err;
80
81 memset(node, 0, sizeof(*node));
82 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
83 size, 0, I915_COLOR_UNEVICTABLE,
84 0, ggtt->mappable_end,
85 DRM_MM_INSERT_LOW);
86
87 mutex_unlock(&ggtt->vm.mutex);
88
89 return err;
90 }
91
92 static void
remove_mappable_node(struct i915_ggtt * ggtt,struct drm_mm_node * node)93 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
94 {
95 mutex_lock(&ggtt->vm.mutex);
96 drm_mm_remove_node(node);
97 mutex_unlock(&ggtt->vm.mutex);
98 }
99
100 int
i915_gem_get_aperture_ioctl(struct drm_device * dev,void * data,struct drm_file * file)101 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
102 struct drm_file *file)
103 {
104 struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
105 struct drm_i915_gem_get_aperture *args = data;
106 struct i915_vma *vma;
107 u64 pinned;
108
109 if (mutex_lock_interruptible(&ggtt->vm.mutex))
110 return -EINTR;
111
112 pinned = ggtt->vm.reserved;
113 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
114 if (i915_vma_is_pinned(vma))
115 pinned += vma->node.size;
116
117 mutex_unlock(&ggtt->vm.mutex);
118
119 args->aper_size = ggtt->vm.total;
120 args->aper_available_size = args->aper_size - pinned;
121
122 return 0;
123 }
124
i915_gem_object_unbind(struct drm_i915_gem_object * obj,unsigned long flags)125 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
126 unsigned long flags)
127 {
128 struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
129 LIST_HEAD(still_in_list);
130 intel_wakeref_t wakeref;
131 struct i915_vma *vma;
132 int ret;
133
134 if (!atomic_read(&obj->bind_count))
135 return 0;
136
137 /*
138 * As some machines use ACPI to handle runtime-resume callbacks, and
139 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
140 * as they are required by the shrinker. Ergo, we wake the device up
141 * first just in case.
142 */
143 wakeref = intel_runtime_pm_get(rpm);
144
145 try_again:
146 ret = 0;
147 spin_lock(&obj->vma.lock);
148 while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
149 struct i915_vma,
150 obj_link))) {
151 struct i915_address_space *vm = vma->vm;
152
153 list_move_tail(&vma->obj_link, &still_in_list);
154 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
155 continue;
156
157 ret = -EAGAIN;
158 if (!i915_vm_tryopen(vm))
159 break;
160
161 /* Prevent vma being freed by i915_vma_parked as we unbind */
162 vma = __i915_vma_get(vma);
163 spin_unlock(&obj->vma.lock);
164
165 if (vma) {
166 ret = -EBUSY;
167 if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
168 !i915_vma_is_active(vma))
169 ret = i915_vma_unbind(vma);
170
171 __i915_vma_put(vma);
172 }
173
174 i915_vm_close(vm);
175 spin_lock(&obj->vma.lock);
176 }
177 list_splice_init(&still_in_list, &obj->vma.list);
178 spin_unlock(&obj->vma.lock);
179
180 if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
181 rcu_barrier(); /* flush the i915_vm_release() */
182 goto try_again;
183 }
184
185 intel_runtime_pm_put(rpm, wakeref);
186
187 return ret;
188 }
189
190 static int
i915_gem_phys_pwrite(struct drm_i915_gem_object * obj,struct drm_i915_gem_pwrite * args,struct drm_file * file)191 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
192 struct drm_i915_gem_pwrite *args,
193 struct drm_file *file)
194 {
195 #ifdef __NetBSD__
196 void *vaddr = obj->mm.u.phys.kva + args->offset;
197 #else
198 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
199 #endif
200 char __user *user_data = u64_to_user_ptr(args->data_ptr);
201
202 /*
203 * We manually control the domain here and pretend that it
204 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
205 */
206 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
207
208 if (copy_from_user(vaddr, user_data, args->size))
209 return -EFAULT;
210
211 drm_clflush_virt_range(vaddr, args->size);
212 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
213
214 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
215 return 0;
216 }
217
218 static int
i915_gem_create(struct drm_file * file,struct intel_memory_region * mr,u64 * size_p,u32 * handle_p)219 i915_gem_create(struct drm_file *file,
220 struct intel_memory_region *mr,
221 u64 *size_p,
222 u32 *handle_p)
223 {
224 struct drm_i915_gem_object *obj;
225 u32 handle;
226 u64 size;
227 int ret;
228
229 GEM_BUG_ON(!is_power_of_2(mr->min_page_size));
230 size = ALIGN(*size_p, mr->min_page_size);
231 if (size == 0)
232 return -EINVAL;
233
234 /* For most of the ABI (e.g. mmap) we think in system pages */
235 GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
236
237 /* Allocate the new object */
238 obj = i915_gem_object_create_region(mr, size, 0);
239 if (IS_ERR(obj))
240 return PTR_ERR(obj);
241
242 ret = drm_gem_handle_create(file, &obj->base, &handle);
243 /* drop reference from allocate - handle holds it now */
244 i915_gem_object_put(obj);
245 if (ret)
246 return ret;
247
248 *handle_p = handle;
249 *size_p = size;
250 return 0;
251 }
252
253 int
i915_gem_dumb_create(struct drm_file * file,struct drm_device * dev,struct drm_mode_create_dumb * args)254 i915_gem_dumb_create(struct drm_file *file,
255 struct drm_device *dev,
256 struct drm_mode_create_dumb *args)
257 {
258 enum intel_memory_type mem_type;
259 int cpp = DIV_ROUND_UP(args->bpp, 8);
260 u32 format;
261
262 switch (cpp) {
263 case 1:
264 format = DRM_FORMAT_C8;
265 break;
266 case 2:
267 format = DRM_FORMAT_RGB565;
268 break;
269 case 4:
270 format = DRM_FORMAT_XRGB8888;
271 break;
272 default:
273 return -EINVAL;
274 }
275
276 /* have to work out size/pitch and return them */
277 args->pitch = ALIGN(args->width * cpp, 64);
278
279 /* align stride to page size so that we can remap */
280 if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format,
281 DRM_FORMAT_MOD_LINEAR))
282 args->pitch = ALIGN(args->pitch, 4096);
283
284 if (args->pitch < args->width)
285 return -EINVAL;
286
287 args->size = mul_u32_u32(args->pitch, args->height);
288
289 mem_type = INTEL_MEMORY_SYSTEM;
290 if (HAS_LMEM(to_i915(dev)))
291 mem_type = INTEL_MEMORY_LOCAL;
292
293 return i915_gem_create(file,
294 intel_memory_region_by_type(to_i915(dev),
295 mem_type),
296 &args->size, &args->handle);
297 }
298
299 /**
300 * Creates a new mm object and returns a handle to it.
301 * @dev: drm device pointer
302 * @data: ioctl data blob
303 * @file: drm file pointer
304 */
305 int
i915_gem_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file)306 i915_gem_create_ioctl(struct drm_device *dev, void *data,
307 struct drm_file *file)
308 {
309 struct drm_i915_private *i915 = to_i915(dev);
310 struct drm_i915_gem_create *args = data;
311
312 i915_gem_flush_free_objects(i915);
313
314 return i915_gem_create(file,
315 intel_memory_region_by_type(i915,
316 INTEL_MEMORY_SYSTEM),
317 &args->size, &args->handle);
318 }
319
320 static int
shmem_pread(struct page * page,int offset,int len,char __user * user_data,bool needs_clflush)321 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
322 bool needs_clflush)
323 {
324 char *vaddr;
325 int ret;
326
327 vaddr = kmap(page);
328
329 if (needs_clflush)
330 drm_clflush_virt_range(vaddr + offset, len);
331
332 ret = __copy_to_user(user_data, vaddr + offset, len);
333
334 kunmap(page);
335
336 return ret ? -EFAULT : 0;
337 }
338
339 static int
i915_gem_shmem_pread(struct drm_i915_gem_object * obj,struct drm_i915_gem_pread * args)340 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
341 struct drm_i915_gem_pread *args)
342 {
343 unsigned int needs_clflush;
344 unsigned int idx, offset;
345 struct dma_fence *fence;
346 char __user *user_data;
347 u64 remain;
348 int ret;
349
350 ret = i915_gem_object_prepare_read(obj, &needs_clflush);
351 if (ret)
352 return ret;
353
354 fence = i915_gem_object_lock_fence(obj);
355 i915_gem_object_finish_access(obj);
356 if (!fence)
357 return -ENOMEM;
358
359 remain = args->size;
360 user_data = u64_to_user_ptr(args->data_ptr);
361 offset = offset_in_page(args->offset);
362 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
363 struct page *page = i915_gem_object_get_page(obj, idx);
364 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
365
366 ret = shmem_pread(page, offset, length, user_data,
367 needs_clflush);
368 if (ret)
369 break;
370
371 remain -= length;
372 user_data += length;
373 offset = 0;
374 }
375
376 i915_gem_object_unlock_fence(obj, fence);
377 return ret;
378 }
379
380 #ifdef __NetBSD__
381 #define __iomem
382 #endif
383 static inline bool
gtt_user_read(struct io_mapping * mapping,loff_t base,int offset,char __user * user_data,int length)384 gtt_user_read(struct io_mapping *mapping,
385 loff_t base, int offset,
386 char __user *user_data, int length)
387 {
388 void __iomem *vaddr;
389 unsigned long unwritten;
390
391 #ifdef __NetBSD__
392 // No fast path for us.
393 unwritten = -EFAULT;
394 #else
395 /* We can use the cpu mem copy function because this is X86. */
396 vaddr = io_mapping_map_atomic_wc(mapping, base);
397 unwritten = __copy_to_user_inatomic(user_data,
398 (void __force *)vaddr + offset,
399 length);
400 io_mapping_unmap_atomic(vaddr);
401 #endif
402 if (unwritten) {
403 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
404 unwritten = copy_to_user(user_data,
405 (void __force *)vaddr + offset,
406 length);
407 #ifdef __NetBSD__
408 io_mapping_unmap(mapping, vaddr, PAGE_SIZE);
409 #else
410 io_mapping_unmap(vaddr);
411 #endif
412 }
413 return unwritten;
414 }
415 #ifdef __NetBSD__
416 #undef __iomem
417 #endif
418
419 static int
i915_gem_gtt_pread(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pread * args)420 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
421 const struct drm_i915_gem_pread *args)
422 {
423 struct drm_i915_private *i915 = to_i915(obj->base.dev);
424 struct i915_ggtt *ggtt = &i915->ggtt;
425 intel_wakeref_t wakeref;
426 struct drm_mm_node node;
427 struct dma_fence *fence;
428 void __user *user_data;
429 struct i915_vma *vma;
430 u64 remain, offset;
431 int ret;
432
433 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
434 vma = ERR_PTR(-ENODEV);
435 if (!i915_gem_object_is_tiled(obj))
436 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
437 PIN_MAPPABLE |
438 PIN_NONBLOCK /* NOWARN */ |
439 PIN_NOEVICT);
440 if (!IS_ERR(vma)) {
441 node.start = i915_ggtt_offset(vma);
442 node.flags = 0;
443 } else {
444 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
445 if (ret)
446 goto out_rpm;
447 GEM_BUG_ON(!drm_mm_node_allocated(&node));
448 }
449
450 ret = i915_gem_object_lock_interruptible(obj);
451 if (ret)
452 goto out_unpin;
453
454 ret = i915_gem_object_set_to_gtt_domain(obj, false);
455 if (ret) {
456 i915_gem_object_unlock(obj);
457 goto out_unpin;
458 }
459
460 fence = i915_gem_object_lock_fence(obj);
461 i915_gem_object_unlock(obj);
462 if (!fence) {
463 ret = -ENOMEM;
464 goto out_unpin;
465 }
466
467 user_data = u64_to_user_ptr(args->data_ptr);
468 remain = args->size;
469 offset = args->offset;
470
471 while (remain > 0) {
472 /* Operation in this page
473 *
474 * page_base = page offset within aperture
475 * page_offset = offset within page
476 * page_length = bytes to copy for this page
477 */
478 u32 page_base = node.start;
479 unsigned page_offset = offset_in_page(offset);
480 unsigned page_length = PAGE_SIZE - page_offset;
481 page_length = remain < page_length ? remain : page_length;
482 if (drm_mm_node_allocated(&node)) {
483 ggtt->vm.insert_page(&ggtt->vm,
484 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
485 node.start, I915_CACHE_NONE, 0);
486 } else {
487 page_base += offset & PAGE_MASK;
488 }
489
490 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
491 user_data, page_length)) {
492 ret = -EFAULT;
493 break;
494 }
495
496 remain -= page_length;
497 user_data += page_length;
498 offset += page_length;
499 }
500
501 i915_gem_object_unlock_fence(obj, fence);
502 out_unpin:
503 if (drm_mm_node_allocated(&node)) {
504 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
505 remove_mappable_node(ggtt, &node);
506 } else {
507 i915_vma_unpin(vma);
508 }
509 out_rpm:
510 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
511 return ret;
512 }
513
514 /**
515 * Reads data from the object referenced by handle.
516 * @dev: drm device pointer
517 * @data: ioctl data blob
518 * @file: drm file pointer
519 *
520 * On error, the contents of *data are undefined.
521 */
522 int
i915_gem_pread_ioctl(struct drm_device * dev,void * data,struct drm_file * file)523 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
524 struct drm_file *file)
525 {
526 struct drm_i915_gem_pread *args = data;
527 struct drm_i915_gem_object *obj;
528 int ret;
529
530 if (args->size == 0)
531 return 0;
532
533 if (!access_ok(u64_to_user_ptr(args->data_ptr),
534 args->size))
535 return -EFAULT;
536
537 obj = i915_gem_object_lookup(file, args->handle);
538 if (!obj)
539 return -ENOENT;
540
541 /* Bounds check source. */
542 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
543 ret = -EINVAL;
544 goto out;
545 }
546
547 trace_i915_gem_object_pread(obj, args->offset, args->size);
548
549 ret = i915_gem_object_wait(obj,
550 I915_WAIT_INTERRUPTIBLE,
551 MAX_SCHEDULE_TIMEOUT);
552 if (ret)
553 goto out;
554
555 ret = i915_gem_object_pin_pages(obj);
556 if (ret)
557 goto out;
558
559 ret = i915_gem_shmem_pread(obj, args);
560 if (ret == -EFAULT || ret == -ENODEV)
561 ret = i915_gem_gtt_pread(obj, args);
562
563 i915_gem_object_unpin_pages(obj);
564 out:
565 i915_gem_object_put(obj);
566 return ret;
567 }
568
569 /* This is the fast write path which cannot handle
570 * page faults in the source data
571 */
572
573 static inline bool
ggtt_write(struct io_mapping * mapping,loff_t base,int offset,char __user * user_data,int length)574 ggtt_write(struct io_mapping *mapping,
575 loff_t base, int offset,
576 char __user *user_data, int length)
577 {
578 #ifdef __NetBSD__
579 return length;
580 #else
581 void __iomem *vaddr;
582 unsigned long unwritten;
583
584 /* We can use the cpu mem copy function because this is X86. */
585 vaddr = io_mapping_map_atomic_wc(mapping, base);
586 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
587 user_data, length);
588 io_mapping_unmap_atomic(vaddr);
589 if (unwritten) {
590 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
591 unwritten = copy_from_user((void __force *)vaddr + offset,
592 user_data, length);
593 io_mapping_unmap(vaddr);
594 }
595
596 return unwritten;
597 #endif
598 }
599
600 /**
601 * This is the fast pwrite path, where we copy the data directly from the
602 * user into the GTT, uncached.
603 * @obj: i915 GEM object
604 * @args: pwrite arguments structure
605 */
606 static int
i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pwrite * args)607 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
608 const struct drm_i915_gem_pwrite *args)
609 {
610 struct drm_i915_private *i915 = to_i915(obj->base.dev);
611 struct i915_ggtt *ggtt = &i915->ggtt;
612 struct intel_runtime_pm *rpm = &i915->runtime_pm;
613 intel_wakeref_t wakeref;
614 struct drm_mm_node node;
615 struct dma_fence *fence;
616 struct i915_vma *vma;
617 u64 remain, offset;
618 void __user *user_data;
619 int ret;
620
621 if (i915_gem_object_has_struct_page(obj)) {
622 /*
623 * Avoid waking the device up if we can fallback, as
624 * waking/resuming is very slow (worst-case 10-100 ms
625 * depending on PCI sleeps and our own resume time).
626 * This easily dwarfs any performance advantage from
627 * using the cache bypass of indirect GGTT access.
628 */
629 wakeref = intel_runtime_pm_get_if_in_use(rpm);
630 if (!wakeref)
631 return -EFAULT;
632 } else {
633 /* No backing pages, no fallback, we must force GGTT access */
634 wakeref = intel_runtime_pm_get(rpm);
635 }
636
637 vma = ERR_PTR(-ENODEV);
638 if (!i915_gem_object_is_tiled(obj))
639 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
640 PIN_MAPPABLE |
641 PIN_NONBLOCK /* NOWARN */ |
642 PIN_NOEVICT);
643 if (!IS_ERR(vma)) {
644 node.start = i915_ggtt_offset(vma);
645 node.flags = 0;
646 } else {
647 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
648 if (ret)
649 goto out_rpm;
650 GEM_BUG_ON(!drm_mm_node_allocated(&node));
651 }
652
653 ret = i915_gem_object_lock_interruptible(obj);
654 if (ret)
655 goto out_unpin;
656
657 ret = i915_gem_object_set_to_gtt_domain(obj, true);
658 if (ret) {
659 i915_gem_object_unlock(obj);
660 goto out_unpin;
661 }
662
663 fence = i915_gem_object_lock_fence(obj);
664 i915_gem_object_unlock(obj);
665 if (!fence) {
666 ret = -ENOMEM;
667 goto out_unpin;
668 }
669
670 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
671
672 user_data = u64_to_user_ptr(args->data_ptr);
673 offset = args->offset;
674 remain = args->size;
675 while (remain) {
676 /* Operation in this page
677 *
678 * page_base = page offset within aperture
679 * page_offset = offset within page
680 * page_length = bytes to copy for this page
681 */
682 u32 page_base = node.start;
683 unsigned int page_offset = offset_in_page(offset);
684 unsigned int page_length = PAGE_SIZE - page_offset;
685 page_length = remain < page_length ? remain : page_length;
686 if (drm_mm_node_allocated(&node)) {
687 /* flush the write before we modify the GGTT */
688 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
689 ggtt->vm.insert_page(&ggtt->vm,
690 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
691 node.start, I915_CACHE_NONE, 0);
692 wmb(); /* flush modifications to the GGTT (insert_page) */
693 } else {
694 page_base += offset & PAGE_MASK;
695 }
696 /* If we get a fault while copying data, then (presumably) our
697 * source page isn't available. Return the error and we'll
698 * retry in the slow path.
699 * If the object is non-shmem backed, we retry again with the
700 * path that handles page fault.
701 */
702 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
703 user_data, page_length)) {
704 ret = -EFAULT;
705 break;
706 }
707
708 remain -= page_length;
709 user_data += page_length;
710 offset += page_length;
711 }
712
713 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
714 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
715
716 i915_gem_object_unlock_fence(obj, fence);
717 out_unpin:
718 if (drm_mm_node_allocated(&node)) {
719 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
720 remove_mappable_node(ggtt, &node);
721 } else {
722 i915_vma_unpin(vma);
723 }
724 out_rpm:
725 intel_runtime_pm_put(rpm, wakeref);
726 return ret;
727 }
728
729 /* Per-page copy function for the shmem pwrite fastpath.
730 * Flushes invalid cachelines before writing to the target if
731 * needs_clflush_before is set and flushes out any written cachelines after
732 * writing if needs_clflush is set.
733 */
734 static int
shmem_pwrite(struct page * page,int offset,int len,char __user * user_data,bool needs_clflush_before,bool needs_clflush_after)735 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
736 bool needs_clflush_before,
737 bool needs_clflush_after)
738 {
739 char *vaddr;
740 int ret;
741
742 vaddr = kmap(page);
743
744 if (needs_clflush_before)
745 drm_clflush_virt_range(vaddr + offset, len);
746
747 ret = __copy_from_user(vaddr + offset, user_data, len);
748 if (!ret && needs_clflush_after)
749 drm_clflush_virt_range(vaddr + offset, len);
750
751 kunmap(page);
752
753 return ret ? -EFAULT : 0;
754 }
755
756 static int
i915_gem_shmem_pwrite(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pwrite * args)757 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
758 const struct drm_i915_gem_pwrite *args)
759 {
760 unsigned int partial_cacheline_write;
761 unsigned int needs_clflush;
762 unsigned int offset, idx;
763 struct dma_fence *fence;
764 void __user *user_data;
765 u64 remain;
766 int ret;
767
768 ret = i915_gem_object_prepare_write(obj, &needs_clflush);
769 if (ret)
770 return ret;
771
772 fence = i915_gem_object_lock_fence(obj);
773 i915_gem_object_finish_access(obj);
774 if (!fence)
775 return -ENOMEM;
776
777 /* If we don't overwrite a cacheline completely we need to be
778 * careful to have up-to-date data by first clflushing. Don't
779 * overcomplicate things and flush the entire patch.
780 */
781 partial_cacheline_write = 0;
782 if (needs_clflush & CLFLUSH_BEFORE)
783 #ifdef __NetBSD__
784 partial_cacheline_write = cpu_info_primary.ci_cflush_lsize - 1;
785 #else
786 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
787 #endif
788
789 user_data = u64_to_user_ptr(args->data_ptr);
790 remain = args->size;
791 offset = offset_in_page(args->offset);
792 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
793 struct page *page = i915_gem_object_get_page(obj, idx);
794 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
795
796 ret = shmem_pwrite(page, offset, length, user_data,
797 (offset | length) & partial_cacheline_write,
798 needs_clflush & CLFLUSH_AFTER);
799 if (ret)
800 break;
801
802 remain -= length;
803 user_data += length;
804 offset = 0;
805 }
806
807 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
808 i915_gem_object_unlock_fence(obj, fence);
809
810 return ret;
811 }
812
813 /**
814 * Writes data to the object referenced by handle.
815 * @dev: drm device
816 * @data: ioctl data blob
817 * @file: drm file
818 *
819 * On error, the contents of the buffer that were to be modified are undefined.
820 */
821 int
i915_gem_pwrite_ioctl(struct drm_device * dev,void * data,struct drm_file * file)822 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
823 struct drm_file *file)
824 {
825 struct drm_i915_gem_pwrite *args = data;
826 struct drm_i915_gem_object *obj;
827 int ret;
828
829 if (args->size == 0)
830 return 0;
831
832 if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
833 return -EFAULT;
834
835 obj = i915_gem_object_lookup(file, args->handle);
836 if (!obj)
837 return -ENOENT;
838
839 /* Bounds check destination. */
840 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
841 ret = -EINVAL;
842 goto err;
843 }
844
845 /* Writes not allowed into this read-only object */
846 if (i915_gem_object_is_readonly(obj)) {
847 ret = -EINVAL;
848 goto err;
849 }
850
851 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
852
853 ret = -ENODEV;
854 if (obj->ops->pwrite)
855 ret = obj->ops->pwrite(obj, args);
856 if (ret != -ENODEV)
857 goto err;
858
859 ret = i915_gem_object_wait(obj,
860 I915_WAIT_INTERRUPTIBLE |
861 I915_WAIT_ALL,
862 MAX_SCHEDULE_TIMEOUT);
863 if (ret)
864 goto err;
865
866 ret = i915_gem_object_pin_pages(obj);
867 if (ret)
868 goto err;
869
870 ret = -EFAULT;
871 /* We can only do the GTT pwrite on untiled buffers, as otherwise
872 * it would end up going through the fenced access, and we'll get
873 * different detiling behavior between reading and writing.
874 * pread/pwrite currently are reading and writing from the CPU
875 * perspective, requiring manual detiling by the client.
876 */
877 if (!i915_gem_object_has_struct_page(obj) ||
878 cpu_write_needs_clflush(obj))
879 /* Note that the gtt paths might fail with non-page-backed user
880 * pointers (e.g. gtt mappings when moving data between
881 * textures). Fallback to the shmem path in that case.
882 */
883 ret = i915_gem_gtt_pwrite_fast(obj, args);
884
885 if (ret == -EFAULT || ret == -ENOSPC) {
886 if (i915_gem_object_has_struct_page(obj))
887 ret = i915_gem_shmem_pwrite(obj, args);
888 else
889 ret = i915_gem_phys_pwrite(obj, args, file);
890 }
891
892 i915_gem_object_unpin_pages(obj);
893 err:
894 i915_gem_object_put(obj);
895 return ret;
896 }
897
898 /**
899 * Called when user space has done writes to this buffer
900 * @dev: drm device
901 * @data: ioctl data blob
902 * @file: drm file
903 */
904 int
i915_gem_sw_finish_ioctl(struct drm_device * dev,void * data,struct drm_file * file)905 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
906 struct drm_file *file)
907 {
908 struct drm_i915_gem_sw_finish *args = data;
909 struct drm_i915_gem_object *obj;
910
911 obj = i915_gem_object_lookup(file, args->handle);
912 if (!obj)
913 return -ENOENT;
914
915 /*
916 * Proxy objects are barred from CPU access, so there is no
917 * need to ban sw_finish as it is a nop.
918 */
919
920 /* Pinned buffers may be scanout, so flush the cache */
921 i915_gem_object_flush_if_display(obj);
922 i915_gem_object_put(obj);
923
924 return 0;
925 }
926
i915_gem_runtime_suspend(struct drm_i915_private * i915)927 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
928 {
929 struct drm_i915_gem_object *obj, *on;
930 int i;
931
932 /*
933 * Only called during RPM suspend. All users of the userfault_list
934 * must be holding an RPM wakeref to ensure that this can not
935 * run concurrently with themselves (and use the struct_mutex for
936 * protection between themselves).
937 */
938
939 list_for_each_entry_safe(obj, on,
940 &i915->ggtt.userfault_list, userfault_link)
941 __i915_gem_object_release_mmap_gtt(obj);
942
943 /*
944 * The fence will be lost when the device powers down. If any were
945 * in use by hardware (i.e. they are pinned), we should not be powering
946 * down! All other fences will be reacquired by the user upon waking.
947 */
948 for (i = 0; i < i915->ggtt.num_fences; i++) {
949 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
950
951 /*
952 * Ideally we want to assert that the fence register is not
953 * live at this point (i.e. that no piece of code will be
954 * trying to write through fence + GTT, as that both violates
955 * our tracking of activity and associated locking/barriers,
956 * but also is illegal given that the hw is powered down).
957 *
958 * Previously we used reg->pin_count as a "liveness" indicator.
959 * That is not sufficient, and we need a more fine-grained
960 * tool if we want to have a sanity check here.
961 */
962
963 if (!reg->vma)
964 continue;
965
966 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
967 reg->dirty = true;
968 }
969 }
970
971 struct i915_vma *
i915_gem_object_ggtt_pin(struct drm_i915_gem_object * obj,const struct i915_ggtt_view * view,u64 size,u64 alignment,u64 flags)972 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
973 const struct i915_ggtt_view *view,
974 u64 size,
975 u64 alignment,
976 u64 flags)
977 {
978 struct drm_i915_private *i915 = to_i915(obj->base.dev);
979 struct i915_ggtt *ggtt = &i915->ggtt;
980 struct i915_vma *vma;
981 int ret;
982
983 if (i915_gem_object_never_bind_ggtt(obj))
984 return ERR_PTR(-ENODEV);
985
986 if (flags & PIN_MAPPABLE &&
987 (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
988 /*
989 * If the required space is larger than the available
990 * aperture, we will not able to find a slot for the
991 * object and unbinding the object now will be in
992 * vain. Worse, doing so may cause us to ping-pong
993 * the object in and out of the Global GTT and
994 * waste a lot of cycles under the mutex.
995 */
996 if (obj->base.size > ggtt->mappable_end)
997 return ERR_PTR(-E2BIG);
998
999 /*
1000 * If NONBLOCK is set the caller is optimistically
1001 * trying to cache the full object within the mappable
1002 * aperture, and *must* have a fallback in place for
1003 * situations where we cannot bind the object. We
1004 * can be a little more lax here and use the fallback
1005 * more often to avoid costly migrations of ourselves
1006 * and other objects within the aperture.
1007 *
1008 * Half-the-aperture is used as a simple heuristic.
1009 * More interesting would to do search for a free
1010 * block prior to making the commitment to unbind.
1011 * That caters for the self-harm case, and with a
1012 * little more heuristics (e.g. NOFAULT, NOEVICT)
1013 * we could try to minimise harm to others.
1014 */
1015 if (flags & PIN_NONBLOCK &&
1016 obj->base.size > ggtt->mappable_end / 2)
1017 return ERR_PTR(-ENOSPC);
1018 }
1019
1020 vma = i915_vma_instance(obj, &ggtt->vm, view);
1021 if (IS_ERR(vma))
1022 return vma;
1023
1024 if (i915_vma_misplaced(vma, size, alignment, flags)) {
1025 if (flags & PIN_NONBLOCK) {
1026 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
1027 return ERR_PTR(-ENOSPC);
1028
1029 if (flags & PIN_MAPPABLE &&
1030 vma->fence_size > ggtt->mappable_end / 2)
1031 return ERR_PTR(-ENOSPC);
1032 }
1033
1034 ret = i915_vma_unbind(vma);
1035 if (ret)
1036 return ERR_PTR(ret);
1037 }
1038
1039 if (vma->fence && !i915_gem_object_is_tiled(obj)) {
1040 mutex_lock(&ggtt->vm.mutex);
1041 ret = i915_vma_revoke_fence(vma);
1042 mutex_unlock(&ggtt->vm.mutex);
1043 if (ret)
1044 return ERR_PTR(ret);
1045 }
1046
1047 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
1048 if (ret)
1049 return ERR_PTR(ret);
1050
1051 return vma;
1052 }
1053
1054 int
i915_gem_madvise_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1055 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
1056 struct drm_file *file_priv)
1057 {
1058 struct drm_i915_private *i915 = to_i915(dev);
1059 struct drm_i915_gem_madvise *args = data;
1060 struct drm_i915_gem_object *obj;
1061 int err;
1062
1063 switch (args->madv) {
1064 case I915_MADV_DONTNEED:
1065 case I915_MADV_WILLNEED:
1066 break;
1067 default:
1068 return -EINVAL;
1069 }
1070
1071 obj = i915_gem_object_lookup(file_priv, args->handle);
1072 if (!obj)
1073 return -ENOENT;
1074
1075 err = mutex_lock_interruptible(&obj->mm.lock);
1076 if (err)
1077 goto out;
1078
1079 if (i915_gem_object_has_pages(obj) &&
1080 i915_gem_object_is_tiled(obj) &&
1081 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1082 if (obj->mm.madv == I915_MADV_WILLNEED) {
1083 GEM_BUG_ON(!obj->mm.quirked);
1084 __i915_gem_object_unpin_pages(obj);
1085 obj->mm.quirked = false;
1086 }
1087 if (args->madv == I915_MADV_WILLNEED) {
1088 GEM_BUG_ON(obj->mm.quirked);
1089 __i915_gem_object_pin_pages(obj);
1090 obj->mm.quirked = true;
1091 }
1092 }
1093
1094 if (obj->mm.madv != __I915_MADV_PURGED)
1095 obj->mm.madv = args->madv;
1096
1097 if (i915_gem_object_has_pages(obj)) {
1098 struct list_head *list;
1099
1100 if (i915_gem_object_is_shrinkable(obj)) {
1101 unsigned long flags;
1102
1103 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1104
1105 if (obj->mm.madv != I915_MADV_WILLNEED)
1106 list = &i915->mm.purge_list;
1107 else
1108 list = &i915->mm.shrink_list;
1109 list_move_tail(&obj->mm.link, list);
1110
1111 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1112 }
1113 }
1114
1115 /* if the object is no longer attached, discard its backing storage */
1116 if (obj->mm.madv == I915_MADV_DONTNEED &&
1117 !i915_gem_object_has_pages(obj))
1118 i915_gem_object_truncate(obj);
1119
1120 args->retained = obj->mm.madv != __I915_MADV_PURGED;
1121 mutex_unlock(&obj->mm.lock);
1122
1123 out:
1124 i915_gem_object_put(obj);
1125 return err;
1126 }
1127
i915_gem_init(struct drm_i915_private * dev_priv)1128 int i915_gem_init(struct drm_i915_private *dev_priv)
1129 {
1130 int ret;
1131
1132 /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1133 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1134 mkwrite_device_info(dev_priv)->page_sizes =
1135 I915_GTT_PAGE_SIZE_4K;
1136
1137 ret = i915_gem_init_userptr(dev_priv);
1138 if (ret)
1139 return ret;
1140
1141 intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1142 intel_wopcm_init(&dev_priv->wopcm);
1143
1144 ret = i915_init_ggtt(dev_priv);
1145 if (ret) {
1146 GEM_BUG_ON(ret == -EIO);
1147 goto err_unlock;
1148 }
1149
1150 /*
1151 * Despite its name intel_init_clock_gating applies both display
1152 * clock gating workarounds; GT mmio workarounds and the occasional
1153 * GT power context workaround. Worse, sometimes it includes a context
1154 * register workaround which we need to apply before we record the
1155 * default HW state for all contexts.
1156 *
1157 * FIXME: break up the workarounds and apply them at the right time!
1158 */
1159 intel_init_clock_gating(dev_priv);
1160
1161 ret = intel_gt_init(&dev_priv->gt);
1162 if (ret)
1163 goto err_unlock;
1164
1165 return 0;
1166
1167 /*
1168 * Unwinding is complicated by that we want to handle -EIO to mean
1169 * disable GPU submission but keep KMS alive. We want to mark the
1170 * HW as irrevisibly wedged, but keep enough state around that the
1171 * driver doesn't explode during runtime.
1172 */
1173 err_unlock:
1174 i915_gem_drain_workqueue(dev_priv);
1175
1176 if (ret != -EIO) {
1177 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1178 i915_gem_cleanup_userptr(dev_priv);
1179 }
1180
1181 if (ret == -EIO) {
1182 /*
1183 * Allow engines or uC initialisation to fail by marking the GPU
1184 * as wedged. But we only want to do this when the GPU is angry,
1185 * for all other failure, such as an allocation failure, bail.
1186 */
1187 if (!intel_gt_is_wedged(&dev_priv->gt)) {
1188 i915_probe_error(dev_priv,
1189 "Failed to initialize GPU, declaring it wedged!\n");
1190 intel_gt_set_wedged(&dev_priv->gt);
1191 }
1192
1193 /* Minimal basic recovery for KMS */
1194 ret = i915_ggtt_enable_hw(dev_priv);
1195 i915_gem_restore_gtt_mappings(dev_priv);
1196 i915_gem_restore_fences(&dev_priv->ggtt);
1197 intel_init_clock_gating(dev_priv);
1198 }
1199
1200 i915_gem_drain_freed_objects(dev_priv);
1201 return ret;
1202 }
1203
i915_gem_driver_register(struct drm_i915_private * i915)1204 void i915_gem_driver_register(struct drm_i915_private *i915)
1205 {
1206 i915_gem_driver_register__shrinker(i915);
1207
1208 intel_engines_driver_register(i915);
1209 }
1210
i915_gem_driver_unregister(struct drm_i915_private * i915)1211 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1212 {
1213 i915_gem_driver_unregister__shrinker(i915);
1214 }
1215
i915_gem_driver_remove(struct drm_i915_private * dev_priv)1216 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1217 {
1218 intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1219
1220 i915_gem_suspend_late(dev_priv);
1221 intel_gt_driver_remove(&dev_priv->gt);
1222 #ifndef __NetBSD__ /* XXX uabi_engines */
1223 dev_priv->uabi_engines = RB_ROOT;
1224 #endif
1225
1226 /* Flush any outstanding unpin_work. */
1227 i915_gem_drain_workqueue(dev_priv);
1228
1229 i915_gem_drain_freed_objects(dev_priv);
1230 }
1231
i915_gem_driver_release(struct drm_i915_private * dev_priv)1232 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1233 {
1234 i915_gem_driver_release__contexts(dev_priv);
1235
1236 intel_gt_driver_release(&dev_priv->gt);
1237
1238 intel_wa_list_free(&dev_priv->gt_wa_list);
1239
1240 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1241 i915_gem_cleanup_userptr(dev_priv);
1242
1243 i915_gem_drain_freed_objects(dev_priv);
1244
1245 WARN_ON(!list_empty(&dev_priv->gem.contexts.list));
1246 }
1247
i915_gem_init__mm(struct drm_i915_private * i915)1248 static void i915_gem_init__mm(struct drm_i915_private *i915)
1249 {
1250 spin_lock_init(&i915->mm.obj_lock);
1251
1252 init_llist_head(&i915->mm.free_list);
1253
1254 INIT_LIST_HEAD(&i915->mm.purge_list);
1255 INIT_LIST_HEAD(&i915->mm.shrink_list);
1256
1257 i915_gem_init__objects(i915);
1258 }
1259
i915_gem_init_early(struct drm_i915_private * dev_priv)1260 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1261 {
1262 i915_gem_init__mm(dev_priv);
1263 i915_gem_init__contexts(dev_priv);
1264
1265 spin_lock_init(&dev_priv->fb_tracking.lock);
1266 }
1267
i915_gem_cleanup_early(struct drm_i915_private * dev_priv)1268 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1269 {
1270 i915_gem_drain_freed_objects(dev_priv);
1271 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1272 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1273 WARN_ON(dev_priv->mm.shrink_count);
1274 spin_lock_destroy(&dev_priv->fb_tracking.lock);
1275 spin_lock_destroy(&dev_priv->mm.obj_lock);
1276 }
1277
i915_gem_freeze(struct drm_i915_private * dev_priv)1278 int i915_gem_freeze(struct drm_i915_private *dev_priv)
1279 {
1280 /* Discard all purgeable objects, let userspace recover those as
1281 * required after resuming.
1282 */
1283 i915_gem_shrink_all(dev_priv);
1284
1285 return 0;
1286 }
1287
i915_gem_freeze_late(struct drm_i915_private * i915)1288 int i915_gem_freeze_late(struct drm_i915_private *i915)
1289 {
1290 struct drm_i915_gem_object *obj;
1291 intel_wakeref_t wakeref;
1292
1293 /*
1294 * Called just before we write the hibernation image.
1295 *
1296 * We need to update the domain tracking to reflect that the CPU
1297 * will be accessing all the pages to create and restore from the
1298 * hibernation, and so upon restoration those pages will be in the
1299 * CPU domain.
1300 *
1301 * To make sure the hibernation image contains the latest state,
1302 * we update that state just before writing out the image.
1303 *
1304 * To try and reduce the hibernation image, we manually shrink
1305 * the objects as well, see i915_gem_freeze()
1306 */
1307
1308 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1309
1310 i915_gem_shrink(i915, -1UL, NULL, ~0);
1311 i915_gem_drain_freed_objects(i915);
1312
1313 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) {
1314 i915_gem_object_lock(obj);
1315 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
1316 i915_gem_object_unlock(obj);
1317 }
1318
1319 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1320
1321 return 0;
1322 }
1323
i915_gem_release(struct drm_device * dev,struct drm_file * file)1324 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
1325 {
1326 struct drm_i915_file_private *file_priv = file->driver_priv;
1327 struct i915_request *request;
1328
1329 /* Clean up our request list when the client is going away, so that
1330 * later retire_requests won't dereference our soon-to-be-gone
1331 * file_priv.
1332 */
1333 spin_lock(&file_priv->mm.lock);
1334 list_for_each_entry(request, &file_priv->mm.request_list, client_link)
1335 request->file_priv = NULL;
1336 spin_unlock(&file_priv->mm.lock);
1337
1338 /*
1339 * XXX This is probably too early -- need to defer with
1340 * callrcu; caller already defers free with kfree_rcu.
1341 */
1342 spin_lock_destroy(&file_priv->mm.lock);
1343 }
1344
i915_gem_open(struct drm_i915_private * i915,struct drm_file * file)1345 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1346 {
1347 struct drm_i915_file_private *file_priv;
1348 int ret;
1349
1350 DRM_DEBUG("\n");
1351
1352 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1353 if (!file_priv)
1354 return -ENOMEM;
1355
1356 file->driver_priv = file_priv;
1357 file_priv->dev_priv = i915;
1358 file_priv->file = file;
1359
1360 spin_lock_init(&file_priv->mm.lock);
1361 INIT_LIST_HEAD(&file_priv->mm.request_list);
1362
1363 file_priv->bsd_engine = -1;
1364 file_priv->hang_timestamp = jiffies;
1365
1366 ret = i915_gem_context_open(i915, file);
1367 if (ret)
1368 kfree(file_priv);
1369
1370 return ret;
1371 }
1372
1373 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1374 #include "selftests/mock_gem_device.c"
1375 #include "selftests/i915_gem.c"
1376 #endif
1377