1 /* $NetBSD: lock.h,v 1.23 2022/04/09 23:43:20 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Wayne Knowles and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Machine-dependent spin lock operations for MIPS processors.
34 *
35 * Note: R2000/R3000 doesn't have any atomic update instructions; this
36 * will cause problems for user applications using this header.
37 */
38
39 #ifndef _MIPS_LOCK_H_
40 #define _MIPS_LOCK_H_
41
42 #include <sys/param.h>
43
44 #include <sys/atomic.h>
45
46 static __inline int
__SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t * __ptr)47 __SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
48 {
49 return *__ptr != __SIMPLELOCK_UNLOCKED;
50 }
51
52 static __inline int
__SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t * __ptr)53 __SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
54 {
55 return *__ptr == __SIMPLELOCK_UNLOCKED;
56 }
57
58 static __inline void
__cpu_simple_lock_clear(__cpu_simple_lock_t * __ptr)59 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
60 {
61 *__ptr = __SIMPLELOCK_UNLOCKED;
62 }
63
64 static __inline void
__cpu_simple_lock_set(__cpu_simple_lock_t * __ptr)65 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
66 {
67 *__ptr = __SIMPLELOCK_LOCKED;
68 }
69
70 #ifndef _HARDKERNEL
71
72 static __inline int
__cpu_simple_lock_try(__cpu_simple_lock_t * lp)73 __cpu_simple_lock_try(__cpu_simple_lock_t *lp)
74 {
75 unsigned long t0, v0;
76
77 __asm volatile(
78 "# -- BEGIN __cpu_simple_lock_try\n"
79 " .set push \n"
80 " .set mips2 \n"
81 "1: ll %0, %4 \n"
82 " bnez %0, 2f \n"
83 " nop \n"
84 " li %0, %3 \n"
85 " sc %0, %2 \n"
86 " beqz %0, 2f \n"
87 " nop \n"
88 " li %1, 1 \n"
89 " sync \n"
90 " j 3f \n"
91 " nop \n"
92 " nop \n"
93 "2: li %1, 0 \n"
94 "3: \n"
95 " .set pop \n"
96 "# -- END __cpu_simple_lock_try \n"
97 : "=r" (t0), "=r" (v0), "+m" (*lp)
98 : "i" (__SIMPLELOCK_LOCKED), "m" (*lp));
99
100 return (v0 != 0);
101 }
102
103 #else /* !_HARDKERNEL */
104
105 u_int _atomic_cas_uint(volatile u_int *, u_int, u_int);
106 u_long _atomic_cas_ulong(volatile u_long *, u_long, u_long);
107 void * _atomic_cas_ptr(volatile void *, void *, void *);
108
109 static __inline int
__cpu_simple_lock_try(__cpu_simple_lock_t * lp)110 __cpu_simple_lock_try(__cpu_simple_lock_t *lp)
111 {
112
113 /*
114 * Successful _atomic_cas_uint functions as a load-acquire --
115 * on MP systems, it issues sync after the LL/SC CAS succeeds;
116 * on non-MP systems every load is a load-acquire so it's moot.
117 * This pairs with the membar_release and store sequence in
118 * __cpu_simple_unlock that functions as a store-release
119 * operation.
120 *
121 * NOTE: This applies only to _atomic_cas_uint (with the
122 * underscore), in sys/arch/mips/mips/lock_stubs_*.S. Not true
123 * for atomic_cas_uint (without the underscore), from
124 * common/lib/libc/arch/mips/atomic/atomic_cas.S which does not
125 * imply a load-acquire. It is unclear why these disagree.
126 */
127 return _atomic_cas_uint(lp,
128 __SIMPLELOCK_UNLOCKED, __SIMPLELOCK_LOCKED) ==
129 __SIMPLELOCK_UNLOCKED;
130 }
131
132 #endif /* _HARDKERNEL */
133
134 static __inline void
__cpu_simple_lock_init(__cpu_simple_lock_t * lp)135 __cpu_simple_lock_init(__cpu_simple_lock_t *lp)
136 {
137
138 *lp = __SIMPLELOCK_UNLOCKED;
139 }
140
141 static __inline void
__cpu_simple_lock(__cpu_simple_lock_t * lp)142 __cpu_simple_lock(__cpu_simple_lock_t *lp)
143 {
144
145 while (!__cpu_simple_lock_try(lp)) {
146 while (*lp == __SIMPLELOCK_LOCKED)
147 /* spin */;
148 }
149 }
150
151 static __inline void
__cpu_simple_unlock(__cpu_simple_lock_t * lp)152 __cpu_simple_unlock(__cpu_simple_lock_t *lp)
153 {
154
155 /*
156 * The membar_release and then store functions as a
157 * store-release operation that pairs with the load-acquire
158 * operation in successful __cpu_simple_lock_try.
159 *
160 * Can't use atomic_store_release here because that's not
161 * available in userland at the moment.
162 */
163 membar_release();
164 *lp = __SIMPLELOCK_UNLOCKED;
165
166 #ifdef _MIPS_ARCH_OCTEONP
167 /*
168 * On Cavium's recommendation, we issue an extra SYNCW that is
169 * not necessary for correct ordering because apparently stores
170 * can get stuck in Octeon store buffers for hundreds of
171 * thousands of cycles, according to the following note:
172 *
173 * Programming Notes:
174 * [...]
175 * Core A (writer)
176 * SW R1, DATA
177 * LI R2, 1
178 * SYNCW
179 * SW R2, FLAG
180 * SYNCW
181 * [...]
182 *
183 * The second SYNCW instruction executed by core A is not
184 * necessary for correctness, but has very important
185 * performance effects on OCTEON. Without it, the store
186 * to FLAG may linger in core A's write buffer before it
187 * becomes visible to other cores. (If core A is not
188 * performing many stores, this may add hundreds of
189 * thousands of cycles to the flag release time since the
190 * OCTEON core normally retains stores to attempt to merge
191 * them before sending the store on the CMB.)
192 * Applications should include this second SYNCW
193 * instruction after flag or lock releases.
194 *
195 * Cavium Networks OCTEON Plus CN50XX Hardware Reference
196 * Manual, July 2008, Appendix A, p. 943.
197 * https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/hactive/CN50XX-HRM-V0.99E.pdf
198 *
199 * XXX It might be prudent to put this into
200 * atomic_store_release itself.
201 */
202 __asm volatile("syncw" ::: "memory");
203 #endif
204 }
205
206 #endif /* _MIPS_LOCK_H_ */
207