1 /*
2 * Copyright 2009-2016 Samy Al Bahra.
3 * Copyright 2013-2016 Olivier Houchard.
4 * Copyright 2016 Alexey Kopytov.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #ifndef CK_PR_AARCH64_LSE_H
30 #define CK_PR_AARCH64_LSE_H
31
32 #ifndef CK_PR_H
33 #error Do not include this file directly, use ck_pr.h
34 #endif
35
36 CK_CC_INLINE static bool
ck_pr_cas_64_2_value(uint64_t target[2],uint64_t compare[2],uint64_t set[2],uint64_t value[2])37 ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2])
38 {
39 uint64_t tmp1;
40 uint64_t tmp2;
41 register uint64_t x0 __asm__ ("x0") = compare[0];
42 register uint64_t x1 __asm__ ("x1") = compare[1];
43 register uint64_t x2 __asm__ ("x2") = set[0];
44 register uint64_t x3 __asm__ ("x3") = set[1];
45
46 __asm__ __volatile__("casp %0, %1, %4, %5, [%6];"
47 "eor %2, %0, %7;"
48 "eor %3, %1, %8;"
49 "orr %2, %2, %3;"
50 : "+&r" (x0), "+&r" (x1), "=&r" (tmp1), "=&r" (tmp2)
51 : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1])
52 : "memory");
53
54 value[0] = x0;
55 value[1] = x1;
56
57 return (!!tmp1);
58 }
59
60 CK_CC_INLINE static bool
ck_pr_cas_ptr_2_value(void * target,void * compare,void * set,void * value)61 ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value)
62 {
63 return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target),
64 CK_CPP_CAST(uint64_t *, compare),
65 CK_CPP_CAST(uint64_t *, set),
66 CK_CPP_CAST(uint64_t *, value)));
67 }
68
69 CK_CC_INLINE static bool
ck_pr_cas_64_2(uint64_t target[2],uint64_t compare[2],uint64_t set[2])70 ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2])
71 {
72 register uint64_t x0 __asm__ ("x0") = compare[0];
73 register uint64_t x1 __asm__ ("x1") = compare[1];
74 register uint64_t x2 __asm__ ("x2") = set[0];
75 register uint64_t x3 __asm__ ("x3") = set[1];
76
77 __asm__ __volatile__("casp %0, %1, %2, %3, [%4];"
78 "eor %0, %0, %5;"
79 "eor %1, %1, %6;"
80 "orr %0, %0, %1;"
81 : "+&r" (x0), "+&r" (x1)
82 : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1])
83 : "memory");
84
85 return (!!x0);
86 }
87 CK_CC_INLINE static bool
ck_pr_cas_ptr_2(void * target,void * compare,void * set)88 ck_pr_cas_ptr_2(void *target, void *compare, void *set)
89 {
90 return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target),
91 CK_CPP_CAST(uint64_t *, compare),
92 CK_CPP_CAST(uint64_t *, set)));
93 }
94
95
96 #define CK_PR_CAS(N, M, T, W, R) \
97 CK_CC_INLINE static bool \
98 ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \
99 { \
100 *(T *)value = compare; \
101 __asm__ __volatile__( \
102 "cas" W " %" R "0, %" R "2, [%1];" \
103 : "+&r" (*(T *)value) \
104 : "r" (target), \
105 "r" (set) \
106 : "memory"); \
107 return (*(T *)value == compare); \
108 } \
109 CK_CC_INLINE static bool \
110 ck_pr_cas_##N(M *target, T compare, T set) \
111 { \
112 T previous = compare; \
113 __asm__ __volatile__( \
114 "cas" W " %" R "0, %" R "2, [%1];" \
115 : "+&r" (previous) \
116 : "r" (target), \
117 "r" (set) \
118 : "memory"); \
119 return (previous == compare); \
120 }
121
122 CK_PR_CAS(ptr, void, void *, "", "")
123
124 #define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R)
125 CK_PR_CAS_S(64, uint64_t, "", "")
126 #ifndef CK_PR_DISABLE_DOUBLE
127 CK_PR_CAS_S(double, double, "", "")
128 #endif
129 CK_PR_CAS_S(32, uint32_t, "", "w")
130 CK_PR_CAS_S(uint, unsigned int, "", "w")
131 CK_PR_CAS_S(int, int, "", "w")
132 CK_PR_CAS_S(16, uint16_t, "h", "w")
133 CK_PR_CAS_S(8, uint8_t, "b", "w")
134 CK_PR_CAS_S(short, short, "h", "w")
135 CK_PR_CAS_S(char, char, "b", "w")
136
137
138 #undef CK_PR_CAS_S
139 #undef CK_PR_CAS
140
141 #define CK_PR_FAS(N, M, T, W, R) \
142 CK_CC_INLINE static T \
143 ck_pr_fas_##N(M *target, T v) \
144 { \
145 T previous; \
146 __asm__ __volatile__( \
147 "swp" W " %" R "2, %" R "0, [%1];" \
148 : "=&r" (previous) \
149 : "r" (target), \
150 "r" (v) \
151 : "memory"); \
152 return (previous); \
153 }
154
155 CK_PR_FAS(64, uint64_t, uint64_t, "", "")
156 CK_PR_FAS(32, uint32_t, uint32_t, "", "w")
157 CK_PR_FAS(ptr, void, void *, "", "")
158 CK_PR_FAS(int, int, int, "", "w")
159 CK_PR_FAS(uint, unsigned int, unsigned int, "", "w")
160 CK_PR_FAS(16, uint16_t, uint16_t, "h", "w")
161 CK_PR_FAS(8, uint8_t, uint8_t, "b", "w")
162 CK_PR_FAS(short, short, short, "h", "w")
163 CK_PR_FAS(char, char, char, "b", "w")
164
165
166 #undef CK_PR_FAS
167
168 #define CK_PR_UNARY(O, N, M, T, I, W, R, S) \
169 CK_CC_INLINE static void \
170 ck_pr_##O##_##N(M *target) \
171 { \
172 __asm__ __volatile__(I ";" \
173 "st" S W " " R "0, [%0];" \
174 : \
175 : "r" (target) \
176 : "x0", "memory"); \
177 return; \
178 }
179
180 CK_PR_UNARY(inc, ptr, void, void *, "mov x0, 1", "", "x", "add")
181 CK_PR_UNARY(dec, ptr, void, void *, "mov x0, -1", "", "x", "add")
182 CK_PR_UNARY(not, ptr, void, void *, "mov x0, -1", "", "x", "eor")
183 CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "mov x0, 1", "", "x", "add")
184 CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "add")
185 CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "eor")
186
187 #define CK_PR_UNARY_S(S, T, W) \
188 CK_PR_UNARY(inc, S, T, T, "mov w0, 1", W, "w", "add") \
189 CK_PR_UNARY(dec, S, T, T, "mov w0, -1", W, "w", "add") \
190 CK_PR_UNARY(not, S, T, T, "mov w0, -1", W, "w", "eor") \
191
192 CK_PR_UNARY_S(32, uint32_t, "")
193 CK_PR_UNARY_S(uint, unsigned int, "")
194 CK_PR_UNARY_S(int, int, "")
195 CK_PR_UNARY_S(16, uint16_t, "h")
196 CK_PR_UNARY_S(8, uint8_t, "b")
197 CK_PR_UNARY_S(short, short, "h")
198 CK_PR_UNARY_S(char, char, "b")
199
200 #undef CK_PR_UNARY_S
201 #undef CK_PR_UNARY
202
203 #define CK_PR_BINARY(O, N, M, T, S, W, R, I) \
204 CK_CC_INLINE static void \
205 ck_pr_##O##_##N(M *target, T delta) \
206 { \
207 __asm__ __volatile__(I ";" \
208 "st" S W " %" R "0, [%1];" \
209 : "+&r" (delta) \
210 : "r" (target) \
211 : "memory"); \
212 return; \
213 }
214
215 CK_PR_BINARY(and, ptr, void, uintptr_t, "clr", "", "", "mvn %0, %0")
216 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "", "")
217 CK_PR_BINARY(or, ptr, void, uintptr_t, "set", "", "", "")
218 CK_PR_BINARY(sub, ptr, void, uintptr_t, "add", "", "", "neg %0, %0")
219 CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "", "")
220 CK_PR_BINARY(and, 64, uint64_t, uint64_t, "clr", "", "", "mvn %0, %0")
221 CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "", "")
222 CK_PR_BINARY(or, 64, uint64_t, uint64_t, "set", "", "", "")
223 CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "add", "", "", "neg %0, %0")
224 CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "", "")
225
226 #define CK_PR_BINARY_S(S, T, W) \
227 CK_PR_BINARY(and, S, T, T, "clr", W, "w", "mvn %w0, %w0") \
228 CK_PR_BINARY(add, S, T, T, "add", W, "w", "") \
229 CK_PR_BINARY(or, S, T, T, "set", W, "w", "") \
230 CK_PR_BINARY(sub, S, T, T, "add", W, "w", "neg %w0, %w0") \
231 CK_PR_BINARY(xor, S, T, T, "eor", W, "w", "")
232
233 CK_PR_BINARY_S(32, uint32_t, "")
234 CK_PR_BINARY_S(uint, unsigned int, "")
235 CK_PR_BINARY_S(int, int, "")
236 CK_PR_BINARY_S(16, uint16_t, "h")
237 CK_PR_BINARY_S(8, uint8_t, "b")
238 CK_PR_BINARY_S(short, short, "h")
239 CK_PR_BINARY_S(char, char, "b")
240
241 #undef CK_PR_BINARY_S
242 #undef CK_PR_BINARY
243
244 CK_CC_INLINE static void *
ck_pr_faa_ptr(void * target,uintptr_t delta)245 ck_pr_faa_ptr(void *target, uintptr_t delta)
246 {
247 uintptr_t previous;
248
249 __asm__ __volatile__(
250 "ldadd %2, %0, [%1];"
251 : "=r" (previous)
252 : "r" (target),
253 "r" (delta)
254 : "memory");
255
256 return (void *)(previous);
257 }
258
259 CK_CC_INLINE static uint64_t
ck_pr_faa_64(uint64_t * target,uint64_t delta)260 ck_pr_faa_64(uint64_t *target, uint64_t delta)
261 {
262 uint64_t previous;
263
264 __asm__ __volatile__(
265 "ldadd %2, %0, [%1];"
266 : "=r" (previous)
267 : "r" (target),
268 "r" (delta)
269 : "memory");
270
271 return (previous);
272 }
273
274 #define CK_PR_FAA(S, T, W) \
275 CK_CC_INLINE static T \
276 ck_pr_faa_##S(T *target, T delta) \
277 { \
278 T previous; \
279 __asm__ __volatile__( \
280 "ldadd" W " %w2, %w0, [%1];" \
281 : "=r" (previous) \
282 : "r" (target), \
283 "r" (delta) \
284 : "memory"); \
285 return (previous); \
286 }
287
288 CK_PR_FAA(32, uint32_t, "")
289 CK_PR_FAA(uint, unsigned int, "")
290 CK_PR_FAA(int, int, "")
291 CK_PR_FAA(16, uint16_t, "h")
292 CK_PR_FAA(8, uint8_t, "b")
293 CK_PR_FAA(short, short, "h")
294 CK_PR_FAA(char, char, "b")
295
296 #undef CK_PR_FAA
297
298 #endif /* CK_PR_AARCH64_LSE_H */
299