1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/kernel.h>
26 #include <asm/fpu/api.h>
27
28 #include "i915_drv.h"
29
30 static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
31
32 #ifdef CONFIG_AS_MOVNTDQA
__memcpy_ntdqa(void * dst,const void * src,unsigned long len)33 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
34 {
35 kernel_fpu_begin();
36
37 len >>= 4;
38 while (len >= 4) {
39 asm("movntdqa (%0), %%xmm0\n"
40 "movntdqa 16(%0), %%xmm1\n"
41 "movntdqa 32(%0), %%xmm2\n"
42 "movntdqa 48(%0), %%xmm3\n"
43 "movaps %%xmm0, (%1)\n"
44 "movaps %%xmm1, 16(%1)\n"
45 "movaps %%xmm2, 32(%1)\n"
46 "movaps %%xmm3, 48(%1)\n"
47 :: "r" (src), "r" (dst) : "memory");
48 src += 64;
49 dst += 64;
50 len -= 4;
51 }
52 while (len--) {
53 asm("movntdqa (%0), %%xmm0\n"
54 "movaps %%xmm0, (%1)\n"
55 :: "r" (src), "r" (dst) : "memory");
56 src += 16;
57 dst += 16;
58 }
59
60 kernel_fpu_end();
61 }
62 #endif
63
64 /**
65 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
66 * @dst: destination pointer
67 * @src: source pointer
68 * @len: how many bytes to copy
69 *
70 * i915_memcpy_from_wc copies @len bytes from @src to @dst using
71 * non-temporal instructions where available. Note that all arguments
72 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
73 * of 16.
74 *
75 * To test whether accelerated reads from WC are supported, use
76 * i915_memcpy_from_wc(NULL, NULL, 0);
77 *
78 * Returns true if the copy was successful, false if the preconditions
79 * are not met.
80 */
i915_memcpy_from_wc(void * dst,const void * src,unsigned long len)81 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
82 {
83 if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
84 return false;
85
86 #ifdef CONFIG_AS_MOVNTDQA
87 if (static_branch_likely(&has_movntdqa)) {
88 if (likely(len))
89 __memcpy_ntdqa(dst, src, len);
90 return true;
91 }
92 #endif
93
94 return false;
95 }
96
i915_memcpy_init_early(struct drm_i915_private * dev_priv)97 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
98 {
99 if (static_cpu_has(X86_FEATURE_XMM4_1))
100 static_branch_enable(&has_movntdqa);
101 }
102