1 /* $NetBSD: vmwgfx_blit.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $ */
2
3 // SPDX-License-Identifier: GPL-2.0 OR MIT
4 /**************************************************************************
5 *
6 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
7 * All Rights Reserved.
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the
11 * "Software"), to deal in the Software without restriction, including
12 * without limitation the rights to use, copy, modify, merge, publish,
13 * distribute, sub license, and/or sell copies of the Software, and to
14 * permit persons to whom the Software is furnished to do so, subject to
15 * the following conditions:
16 *
17 * The above copyright notice and this permission notice (including the
18 * next paragraph) shall be included in all copies or substantial portions
19 * of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
24 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
25 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
26 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
27 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 *
29 **************************************************************************/
30
31 #include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: vmwgfx_blit.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $");
33
34 #include "vmwgfx_drv.h"
35
36 /*
37 * Template that implements find_first_diff() for a generic
38 * unsigned integer type. @size and return value are in bytes.
39 */
40 #define VMW_FIND_FIRST_DIFF(_type) \
41 static size_t vmw_find_first_diff_ ## _type \
42 (const _type * dst, const _type * src, size_t size)\
43 { \
44 size_t i; \
45 \
46 for (i = 0; i < size; i += sizeof(_type)) { \
47 if (*dst++ != *src++) \
48 break; \
49 } \
50 \
51 return i; \
52 }
53
54
55 /*
56 * Template that implements find_last_diff() for a generic
57 * unsigned integer type. Pointers point to the item following the
58 * *end* of the area to be examined. @size and return value are in
59 * bytes.
60 */
61 #define VMW_FIND_LAST_DIFF(_type) \
62 static ssize_t vmw_find_last_diff_ ## _type( \
63 const _type * dst, const _type * src, size_t size) \
64 { \
65 while (size) { \
66 if (*--dst != *--src) \
67 break; \
68 \
69 size -= sizeof(_type); \
70 } \
71 return size; \
72 }
73
74
75 /*
76 * Instantiate find diff functions for relevant unsigned integer sizes,
77 * assuming that wider integers are faster (including aligning) up to the
78 * architecture native width, which is assumed to be 32 bit unless
79 * CONFIG_64BIT is defined.
80 */
81 VMW_FIND_FIRST_DIFF(u8);
82 VMW_FIND_LAST_DIFF(u8);
83
84 VMW_FIND_FIRST_DIFF(u16);
85 VMW_FIND_LAST_DIFF(u16);
86
87 VMW_FIND_FIRST_DIFF(u32);
88 VMW_FIND_LAST_DIFF(u32);
89
90 #ifdef CONFIG_64BIT
91 VMW_FIND_FIRST_DIFF(u64);
92 VMW_FIND_LAST_DIFF(u64);
93 #endif
94
95
96 /* We use size aligned copies. This computes (addr - align(addr)) */
97 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
98
99
100 /*
101 * Template to compute find_first_diff() for a certain integer type
102 * including a head copy for alignment, and adjustment of parameters
103 * for tail find or increased resolution find using an unsigned integer find
104 * of smaller width. If finding is complete, and resolution is sufficient,
105 * the macro executes a return statement. Otherwise it falls through.
106 */
107 #define VMW_TRY_FIND_FIRST_DIFF(_type) \
108 do { \
109 unsigned int spill = SPILL(dst, _type); \
110 size_t diff_offs; \
111 \
112 if (spill && spill == SPILL(src, _type) && \
113 sizeof(_type) - spill <= size) { \
114 spill = sizeof(_type) - spill; \
115 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \
116 if (diff_offs < spill) \
117 return round_down(offset + diff_offs, granularity); \
118 \
119 dst += spill; \
120 src += spill; \
121 size -= spill; \
122 offset += spill; \
123 spill = 0; \
124 } \
125 if (!spill && !SPILL(src, _type)) { \
126 size_t to_copy = size & ~(sizeof(_type) - 1); \
127 \
128 diff_offs = vmw_find_first_diff_ ## _type \
129 ((_type *) dst, (_type *) src, to_copy); \
130 if (diff_offs >= size || granularity == sizeof(_type)) \
131 return (offset + diff_offs); \
132 \
133 dst += diff_offs; \
134 src += diff_offs; \
135 size -= diff_offs; \
136 offset += diff_offs; \
137 } \
138 } while (0) \
139
140
141 /**
142 * vmw_find_first_diff - find the first difference between dst and src
143 *
144 * @dst: The destination address
145 * @src: The source address
146 * @size: Number of bytes to compare
147 * @granularity: The granularity needed for the return value in bytes.
148 * return: The offset from find start where the first difference was
149 * encountered in bytes. If no difference was found, the function returns
150 * a value >= @size.
151 */
vmw_find_first_diff(const u8 * dst,const u8 * src,size_t size,size_t granularity)152 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
153 size_t granularity)
154 {
155 size_t offset = 0;
156
157 /*
158 * Try finding with large integers if alignment allows, or we can
159 * fix it. Fall through if we need better resolution or alignment
160 * was bad.
161 */
162 #ifdef CONFIG_64BIT
163 VMW_TRY_FIND_FIRST_DIFF(u64);
164 #endif
165 VMW_TRY_FIND_FIRST_DIFF(u32);
166 VMW_TRY_FIND_FIRST_DIFF(u16);
167
168 return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
169 granularity);
170 }
171
172
173 /*
174 * Template to compute find_last_diff() for a certain integer type
175 * including a tail copy for alignment, and adjustment of parameters
176 * for head find or increased resolution find using an unsigned integer find
177 * of smaller width. If finding is complete, and resolution is sufficient,
178 * the macro executes a return statement. Otherwise it falls through.
179 */
180 #define VMW_TRY_FIND_LAST_DIFF(_type) \
181 do { \
182 unsigned int spill = SPILL(dst, _type); \
183 ssize_t location; \
184 ssize_t diff_offs; \
185 \
186 if (spill && spill <= size && spill == SPILL(src, _type)) { \
187 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
188 if (diff_offs) { \
189 location = size - spill + diff_offs - 1; \
190 return round_down(location, granularity); \
191 } \
192 \
193 dst -= spill; \
194 src -= spill; \
195 size -= spill; \
196 spill = 0; \
197 } \
198 if (!spill && !SPILL(src, _type)) { \
199 size_t to_copy = round_down(size, sizeof(_type)); \
200 \
201 diff_offs = vmw_find_last_diff_ ## _type \
202 ((_type *) dst, (_type *) src, to_copy); \
203 location = size - to_copy + diff_offs - sizeof(_type); \
204 if (location < 0 || granularity == sizeof(_type)) \
205 return location; \
206 \
207 dst -= to_copy - diff_offs; \
208 src -= to_copy - diff_offs; \
209 size -= to_copy - diff_offs; \
210 } \
211 } while (0)
212
213
214 /**
215 * vmw_find_last_diff - find the last difference between dst and src
216 *
217 * @dst: The destination address
218 * @src: The source address
219 * @size: Number of bytes to compare
220 * @granularity: The granularity needed for the return value in bytes.
221 * return: The offset from find start where the last difference was
222 * encountered in bytes, or a negative value if no difference was found.
223 */
vmw_find_last_diff(const u8 * dst,const u8 * src,size_t size,size_t granularity)224 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
225 size_t granularity)
226 {
227 dst += size;
228 src += size;
229
230 #ifdef CONFIG_64BIT
231 VMW_TRY_FIND_LAST_DIFF(u64);
232 #endif
233 VMW_TRY_FIND_LAST_DIFF(u32);
234 VMW_TRY_FIND_LAST_DIFF(u16);
235
236 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
237 granularity);
238 }
239
240
241 /**
242 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
243 * struct vmw_diff_cpy.
244 *
245 * @diff: The struct vmw_diff_cpy closure argument (unused).
246 * @dest: The copy destination.
247 * @src: The copy source.
248 * @n: Number of bytes to copy.
249 */
vmw_memcpy(struct vmw_diff_cpy * diff,u8 * dest,const u8 * src,size_t n)250 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
251 {
252 memcpy(dest, src, n);
253 }
254
255
256 /**
257 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
258 *
259 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
260 * @diff_offs: The offset from @diff->line_offset where the difference was
261 * found.
262 */
vmw_adjust_rect(struct vmw_diff_cpy * diff,size_t diff_offs)263 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
264 {
265 size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
266 struct drm_rect *rect = &diff->rect;
267
268 rect->x1 = min_t(int, rect->x1, offs);
269 rect->x2 = max_t(int, rect->x2, offs + 1);
270 rect->y1 = min_t(int, rect->y1, diff->line);
271 rect->y2 = max_t(int, rect->y2, diff->line + 1);
272 }
273
274 /**
275 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
276 *
277 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
278 * @dest: The copy destination.
279 * @src: The copy source.
280 * @n: Number of bytes to copy.
281 *
282 * In order to correctly track the modified content, the field @diff->line must
283 * be pre-loaded with the current line number, the field @diff->line_offset must
284 * be pre-loaded with the line offset in bytes where the copy starts, and
285 * finally the field @diff->cpp need to be preloaded with the number of bytes
286 * per unit in the horizontal direction of the area we're examining.
287 * Typically bytes per pixel.
288 * This is needed to know the needed granularity of the difference computing
289 * operations. A higher cpp generally leads to faster execution at the cost of
290 * bounding box width precision.
291 */
vmw_diff_memcpy(struct vmw_diff_cpy * diff,u8 * dest,const u8 * src,size_t n)292 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
293 size_t n)
294 {
295 ssize_t csize, byte_len;
296
297 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
298 return;
299
300 /* TODO: Possibly use a single vmw_find_first_diff per line? */
301 csize = vmw_find_first_diff(dest, src, n, diff->cpp);
302 if (csize < n) {
303 vmw_adjust_rect(diff, csize);
304 byte_len = diff->cpp;
305
306 /*
307 * Starting from where first difference was found, find
308 * location of last difference, and then copy.
309 */
310 diff->line_offset += csize;
311 dest += csize;
312 src += csize;
313 n -= csize;
314 csize = vmw_find_last_diff(dest, src, n, diff->cpp);
315 if (csize >= 0) {
316 byte_len += csize;
317 vmw_adjust_rect(diff, csize);
318 }
319 memcpy(dest, src, byte_len);
320 }
321 diff->line_offset += n;
322 }
323
324 /**
325 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
326 *
327 * @mapped_dst: Already mapped destination page index in @dst_pages.
328 * @dst_addr: Kernel virtual address of mapped destination page.
329 * @dst_pages: Array of destination bo pages.
330 * @dst_num_pages: Number of destination bo pages.
331 * @dst_prot: Destination bo page protection.
332 * @mapped_src: Already mapped source page index in @dst_pages.
333 * @src_addr: Kernel virtual address of mapped source page.
334 * @src_pages: Array of source bo pages.
335 * @src_num_pages: Number of source bo pages.
336 * @src_prot: Source bo page protection.
337 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
338 */
339 struct vmw_bo_blit_line_data {
340 u32 mapped_dst;
341 u8 *dst_addr;
342 struct page **dst_pages;
343 u32 dst_num_pages;
344 pgprot_t dst_prot;
345 u32 mapped_src;
346 u8 *src_addr;
347 struct page **src_pages;
348 u32 src_num_pages;
349 pgprot_t src_prot;
350 struct vmw_diff_cpy *diff;
351 };
352
353 /**
354 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
355 *
356 * @d: Blit data as described above.
357 * @dst_offset: Destination copy start offset from start of bo.
358 * @src_offset: Source copy start offset from start of bo.
359 * @bytes_to_copy: Number of bytes to copy in this line.
360 */
vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data * d,u32 dst_offset,u32 src_offset,u32 bytes_to_copy)361 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
362 u32 dst_offset,
363 u32 src_offset,
364 u32 bytes_to_copy)
365 {
366 struct vmw_diff_cpy *diff = d->diff;
367
368 while (bytes_to_copy) {
369 u32 copy_size = bytes_to_copy;
370 u32 dst_page = dst_offset >> PAGE_SHIFT;
371 u32 src_page = src_offset >> PAGE_SHIFT;
372 u32 dst_page_offset = dst_offset & ~PAGE_MASK;
373 u32 src_page_offset = src_offset & ~PAGE_MASK;
374 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
375 bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
376 unmap_dst);
377
378 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
379 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
380
381 if (unmap_src) {
382 ttm_kunmap_atomic_prot(d->src_addr, d->src_prot);
383 d->src_addr = NULL;
384 }
385
386 if (unmap_dst) {
387 ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot);
388 d->dst_addr = NULL;
389 }
390
391 if (!d->dst_addr) {
392 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
393 return -EINVAL;
394
395 d->dst_addr =
396 ttm_kmap_atomic_prot(d->dst_pages[dst_page],
397 d->dst_prot);
398 if (!d->dst_addr)
399 return -ENOMEM;
400
401 d->mapped_dst = dst_page;
402 }
403
404 if (!d->src_addr) {
405 if (WARN_ON_ONCE(src_page >= d->src_num_pages))
406 return -EINVAL;
407
408 d->src_addr =
409 ttm_kmap_atomic_prot(d->src_pages[src_page],
410 d->src_prot);
411 if (!d->src_addr)
412 return -ENOMEM;
413
414 d->mapped_src = src_page;
415 }
416 diff->do_cpy(diff, d->dst_addr + dst_page_offset,
417 d->src_addr + src_page_offset, copy_size);
418
419 bytes_to_copy -= copy_size;
420 dst_offset += copy_size;
421 src_offset += copy_size;
422 }
423
424 return 0;
425 }
426
427 /**
428 * ttm_bo_cpu_blit - in-kernel cpu blit.
429 *
430 * @dst: Destination buffer object.
431 * @dst_offset: Destination offset of blit start in bytes.
432 * @dst_stride: Destination stride in bytes.
433 * @src: Source buffer object.
434 * @src_offset: Source offset of blit start in bytes.
435 * @src_stride: Source stride in bytes.
436 * @w: Width of blit.
437 * @h: Height of blit.
438 * return: Zero on success. Negative error value on failure. Will print out
439 * kernel warnings on caller bugs.
440 *
441 * Performs a CPU blit from one buffer object to another avoiding a full
442 * bo vmap which may exhaust- or fragment vmalloc space.
443 * On supported architectures (x86), we're using kmap_atomic which avoids
444 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
445 * reference already set-up mappings.
446 *
447 * Neither of the buffer objects may be placed in PCI memory
448 * (Fixed memory in TTM terminology) when using this function.
449 */
vmw_bo_cpu_blit(struct ttm_buffer_object * dst,u32 dst_offset,u32 dst_stride,struct ttm_buffer_object * src,u32 src_offset,u32 src_stride,u32 w,u32 h,struct vmw_diff_cpy * diff)450 int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
451 u32 dst_offset, u32 dst_stride,
452 struct ttm_buffer_object *src,
453 u32 src_offset, u32 src_stride,
454 u32 w, u32 h,
455 struct vmw_diff_cpy *diff)
456 {
457 struct ttm_operation_ctx ctx = {
458 .interruptible = false,
459 .no_wait_gpu = false
460 };
461 u32 j, initial_line = dst_offset / dst_stride;
462 struct vmw_bo_blit_line_data d;
463 int ret = 0;
464
465 /* Buffer objects need to be either pinned or reserved: */
466 if (!(dst->mem.placement & TTM_PL_FLAG_NO_EVICT))
467 dma_resv_assert_held(dst->base.resv);
468 if (!(src->mem.placement & TTM_PL_FLAG_NO_EVICT))
469 dma_resv_assert_held(src->base.resv);
470
471 if (dst->ttm->state == tt_unpopulated) {
472 ret = dst->ttm->bdev->driver->ttm_tt_populate(dst->ttm, &ctx);
473 if (ret)
474 return ret;
475 }
476
477 if (src->ttm->state == tt_unpopulated) {
478 ret = src->ttm->bdev->driver->ttm_tt_populate(src->ttm, &ctx);
479 if (ret)
480 return ret;
481 }
482
483 d.mapped_dst = 0;
484 d.mapped_src = 0;
485 d.dst_addr = NULL;
486 d.src_addr = NULL;
487 d.dst_pages = dst->ttm->pages;
488 d.src_pages = src->ttm->pages;
489 d.dst_num_pages = dst->num_pages;
490 d.src_num_pages = src->num_pages;
491 d.dst_prot = ttm_io_prot(dst->mem.placement, PAGE_KERNEL);
492 d.src_prot = ttm_io_prot(src->mem.placement, PAGE_KERNEL);
493 d.diff = diff;
494
495 for (j = 0; j < h; ++j) {
496 diff->line = j + initial_line;
497 diff->line_offset = dst_offset % dst_stride;
498 ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w);
499 if (ret)
500 goto out;
501
502 dst_offset += dst_stride;
503 src_offset += src_stride;
504 }
505 out:
506 if (d.src_addr)
507 ttm_kunmap_atomic_prot(d.src_addr, d.src_prot);
508 if (d.dst_addr)
509 ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot);
510
511 return ret;
512 }
513