1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020 Intel Corporation
4 */
5
6 #include <linux/iosys-map.h>
7 #include <linux/mm.h>
8 #include <linux/pagemap.h>
9 #include <linux/shmem_fs.h>
10 #include <linux/vmalloc.h>
11
12 #include "i915_drv.h"
13 #include "gem/i915_gem_object.h"
14 #include "gem/i915_gem_lmem.h"
15 #include "shmem_utils.h"
16
17 #ifdef __linux__
18
shmem_create_from_data(const char * name,void * data,size_t len)19 struct file *shmem_create_from_data(const char *name, void *data, size_t len)
20 {
21 struct file *file;
22 int err;
23
24 file = shmem_file_setup(name, PAGE_ALIGN(len), VM_NORESERVE);
25 if (IS_ERR(file))
26 return file;
27
28 err = shmem_write(file, 0, data, len);
29 if (err) {
30 fput(file);
31 return ERR_PTR(err);
32 }
33
34 return file;
35 }
36
shmem_create_from_object(struct drm_i915_gem_object * obj)37 struct file *shmem_create_from_object(struct drm_i915_gem_object *obj)
38 {
39 enum i915_map_type map_type;
40 struct file *file;
41 void *ptr;
42
43 if (i915_gem_object_is_shmem(obj)) {
44 file = obj->base.filp;
45 atomic_long_inc(&file->f_count);
46 return file;
47 }
48
49 map_type = i915_gem_object_is_lmem(obj) ? I915_MAP_WC : I915_MAP_WB;
50 ptr = i915_gem_object_pin_map_unlocked(obj, map_type);
51 if (IS_ERR(ptr))
52 return ERR_CAST(ptr);
53
54 file = shmem_create_from_data("", ptr, obj->base.size);
55 i915_gem_object_unpin_map(obj);
56
57 return file;
58 }
59
shmem_pin_map(struct file * file)60 void *shmem_pin_map(struct file *file)
61 {
62 struct page **pages;
63 size_t n_pages, i;
64 void *vaddr;
65
66 n_pages = file->f_mapping->host->i_size >> PAGE_SHIFT;
67 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
68 if (!pages)
69 return NULL;
70
71 for (i = 0; i < n_pages; i++) {
72 pages[i] = shmem_read_mapping_page_gfp(file->f_mapping, i,
73 GFP_KERNEL);
74 if (IS_ERR(pages[i]))
75 goto err_page;
76 }
77
78 vaddr = vmap(pages, n_pages, VM_MAP_PUT_PAGES, PAGE_KERNEL);
79 if (!vaddr)
80 goto err_page;
81 mapping_set_unevictable(file->f_mapping);
82 return vaddr;
83 err_page:
84 while (i--)
85 put_page(pages[i]);
86 kvfree(pages);
87 return NULL;
88 }
89
shmem_unpin_map(struct file * file,void * ptr)90 void shmem_unpin_map(struct file *file, void *ptr)
91 {
92 mapping_clear_unevictable(file->f_mapping);
93 vfree(ptr);
94 }
95
__shmem_rw(struct file * file,loff_t off,void * ptr,size_t len,bool write)96 static int __shmem_rw(struct file *file, loff_t off,
97 void *ptr, size_t len,
98 bool write)
99 {
100 unsigned long pfn;
101
102 for (pfn = off >> PAGE_SHIFT; len; pfn++) {
103 unsigned int this =
104 min_t(size_t, PAGE_SIZE - offset_in_page(off), len);
105 struct page *page;
106 void *vaddr;
107
108 page = shmem_read_mapping_page_gfp(file->f_mapping, pfn,
109 GFP_KERNEL);
110 if (IS_ERR(page))
111 return PTR_ERR(page);
112
113 vaddr = kmap(page);
114 if (write) {
115 memcpy(vaddr + offset_in_page(off), ptr, this);
116 set_page_dirty(page);
117 } else {
118 memcpy(ptr, vaddr + offset_in_page(off), this);
119 }
120 mark_page_accessed(page);
121 kunmap(page);
122 put_page(page);
123
124 len -= this;
125 ptr += this;
126 off = 0;
127 }
128
129 return 0;
130 }
131
shmem_read_to_iosys_map(struct file * file,loff_t off,struct iosys_map * map,size_t map_off,size_t len)132 int shmem_read_to_iosys_map(struct file *file, loff_t off,
133 struct iosys_map *map, size_t map_off, size_t len)
134 {
135 unsigned long pfn;
136
137 for (pfn = off >> PAGE_SHIFT; len; pfn++) {
138 unsigned int this =
139 min_t(size_t, PAGE_SIZE - offset_in_page(off), len);
140 struct page *page;
141 void *vaddr;
142
143 page = shmem_read_mapping_page_gfp(file->f_mapping, pfn,
144 GFP_KERNEL);
145 if (IS_ERR(page))
146 return PTR_ERR(page);
147
148 vaddr = kmap(page);
149 iosys_map_memcpy_to(map, map_off, vaddr + offset_in_page(off),
150 this);
151 mark_page_accessed(page);
152 kunmap(page);
153 put_page(page);
154
155 len -= this;
156 map_off += this;
157 off = 0;
158 }
159
160 return 0;
161 }
162
shmem_read(struct file * file,loff_t off,void * dst,size_t len)163 int shmem_read(struct file *file, loff_t off, void *dst, size_t len)
164 {
165 return __shmem_rw(file, off, dst, len, false);
166 }
167
shmem_write(struct file * file,loff_t off,void * src,size_t len)168 int shmem_write(struct file *file, loff_t off, void *src, size_t len)
169 {
170 return __shmem_rw(file, off, src, len, true);
171 }
172
173 #endif /* __linux__ */
174
175 struct uvm_object *
uao_create_from_data(const char * name,void * data,size_t len)176 uao_create_from_data(const char *name, void *data, size_t len)
177 {
178 struct uvm_object *uao;
179 int err;
180
181 uao = uao_create(PAGE_ALIGN(len), 0);
182 if (uao == NULL) {
183 return ERR_PTR(-ENOMEM);
184 }
185
186 err = uao_write(uao, 0, data, len);
187 if (err) {
188 uao_detach(uao);
189 return ERR_PTR(err);
190 }
191
192 return uao;
193 }
194
195 struct uvm_object *
uao_create_from_object(struct drm_i915_gem_object * obj)196 uao_create_from_object(struct drm_i915_gem_object *obj)
197 {
198 struct uvm_object *uao;
199 void *ptr;
200
201 if (i915_gem_object_is_shmem(obj)) {
202 uao_reference(obj->base.uao);
203 return obj->base.uao;
204 }
205
206 ptr = i915_gem_object_pin_map_unlocked(obj, i915_gem_object_is_lmem(obj) ?
207 I915_MAP_WC : I915_MAP_WB);
208 if (IS_ERR(ptr))
209 return ERR_CAST(ptr);
210
211 uao = uao_create_from_data("", ptr, obj->base.size);
212 i915_gem_object_unpin_map(obj);
213
214 return uao;
215 }
216
__uao_rw(struct uvm_object * uao,loff_t off,void * ptr,size_t len,bool write)217 static int __uao_rw(struct uvm_object *uao, loff_t off,
218 void *ptr, size_t len,
219 bool write)
220 {
221 struct pglist plist;
222 struct vm_page *page;
223 vaddr_t pgoff = trunc_page(off);
224 size_t olen = round_page(len);
225
226 TAILQ_INIT(&plist);
227 if (uvm_obj_wire(uao, pgoff, olen, &plist))
228 return -ENOMEM;
229
230 TAILQ_FOREACH(page, &plist, pageq) {
231 unsigned int this =
232 min_t(size_t, PAGE_SIZE - offset_in_page(off), len);
233 void *vaddr = kmap(page);
234
235 if (write) {
236 memcpy(vaddr + offset_in_page(off), ptr, this);
237 set_page_dirty(page);
238 } else {
239 memcpy(ptr, vaddr + offset_in_page(off), this);
240 }
241
242 kunmap_va(vaddr);
243 len -= this;
244 ptr += this;
245 off = 0;
246 }
247
248 uvm_obj_unwire(uao, pgoff, olen);
249
250 return 0;
251 }
252
uao_read_to_iosys_map(struct uvm_object * uao,loff_t off,struct iosys_map * map,size_t map_off,size_t len)253 int uao_read_to_iosys_map(struct uvm_object *uao, loff_t off,
254 struct iosys_map *map, size_t map_off, size_t len)
255 {
256 struct pglist plist;
257 struct vm_page *page;
258 vaddr_t pgoff = trunc_page(off);
259 size_t olen = round_page(len);
260
261 TAILQ_INIT(&plist);
262 if (uvm_obj_wire(uao, pgoff, olen, &plist))
263 return -ENOMEM;
264
265 TAILQ_FOREACH(page, &plist, pageq) {
266 unsigned int this =
267 min_t(size_t, PAGE_SIZE - offset_in_page(off), len);
268 void *vaddr;
269
270 vaddr = kmap(page);
271 iosys_map_memcpy_to(map, map_off, vaddr + offset_in_page(off),
272 this);
273 kunmap_va(vaddr);
274
275 len -= this;
276 map_off += this;
277 off = 0;
278 }
279
280 uvm_obj_unwire(uao, pgoff, olen);
281
282 return 0;
283 }
284
uao_read(struct uvm_object * uao,loff_t off,void * dst,size_t len)285 int uao_read(struct uvm_object *uao, loff_t off, void *dst, size_t len)
286 {
287 return __uao_rw(uao, off, dst, len, false);
288 }
289
uao_write(struct uvm_object * uao,loff_t off,void * src,size_t len)290 int uao_write(struct uvm_object *uao, loff_t off, void *src, size_t len)
291 {
292 return __uao_rw(uao, off, src, len, true);
293 }
294
295 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
296 #include "st_shmem_utils.c"
297 #endif
298