1 /* $NetBSD: lockstat.h,v 1.15 2022/02/27 14:16:12 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #ifndef _SYS_LOCKSTAT_H_ 33 #define _SYS_LOCKSTAT_H_ 34 35 #ifdef _KERNEL_OPT 36 #include "opt_dtrace.h" 37 #include <lockstat.h> 38 #endif 39 40 #include <sys/types.h> 41 42 #include <sys/ioccom.h> 43 #include <sys/lock.h> 44 #include <sys/queue.h> 45 #include <sys/time.h> 46 47 #if defined(_KERNEL) && defined(__HAVE_CPU_COUNTER) 48 #include <machine/cpu_counter.h> 49 #endif 50 51 /* 52 * Interface version. The interface is not designed to provide 53 * compatibility across NetBSD releases. 54 */ 55 56 #define IOC_LOCKSTAT_GVERSION _IOR('L', 0, int) 57 58 #define LS_VERSION 5 59 60 /* 61 * Enable request. We can limit tracing by the call site and by 62 * the lock. We also specify the number of event buffers to 63 * allocate up front, and what kind of events to track. 64 */ 65 66 #define IOC_LOCKSTAT_ENABLE _IOW('L', 1, lsenable_t) 67 68 #define LE_CALLSITE 0x01 /* track call sites */ 69 #define LE_ONE_CALLSITE 0x02 /* specific call site */ 70 #define LE_ONE_LOCK 0x04 /* specific lock */ 71 #define LE_LOCK 0x08 /* track locks */ 72 73 typedef struct lsenable { 74 uintptr_t le_csstart; /* callsite start */ 75 uintptr_t le_csend; /* callsite end */ 76 uintptr_t le_lockstart; /* lock address start */ 77 uintptr_t le_lockend; /* lock address end */ 78 uintptr_t le_nbufs; /* buffers to allocate, 0 = default */ 79 u_int le_flags; /* request flags */ 80 u_int le_mask; /* event mask (LB_*) */ 81 } lsenable_t; 82 83 /* 84 * Disable request. 85 */ 86 87 #define IOC_LOCKSTAT_DISABLE _IOR('L', 2, lsdisable_t) 88 89 typedef struct lsdisable { 90 size_t ld_size; /* buffer space allocated */ 91 struct timespec ld_time; /* time spent enabled */ 92 uint64_t ld_freq[64]; /* counter HZ by CPU number */ 93 } lsdisable_t; 94 95 /* 96 * Event buffers returned from reading from the devices. 97 */ 98 99 /* 100 * Event types, for lockstat_event(). Stored in lb_flags but should be 101 * meaningless to the consumer, also provided with the enable request 102 * in le_mask. 103 */ 104 #define LB_SPIN 0x00000001 105 #define LB_SLEEP1 0x00000002 106 #define LB_SLEEP2 0x00000003 107 #define LB_NEVENT 0x00000003 108 #define LB_EVENT_MASK 0x000000ff 109 110 /* 111 * Lock types, the only part of lb_flags that should be inspected. Also 112 * provided with the enable request in le_mask. 113 */ 114 #define LB_ADAPTIVE_MUTEX 0x00000100 115 #define LB_SPIN_MUTEX 0x00000200 116 #define LB_RWLOCK 0x00000300 117 #define LB_NOPREEMPT 0x00000400 118 #define LB_KERNEL_LOCK 0x00000500 119 #define LB_MISC 0x00000600 120 #define LB_NLOCK 0x00000600 121 #define LB_LOCK_MASK 0x0000ff00 122 #define LB_LOCK_SHIFT 8 123 124 #define LB_DTRACE 0x00010000 125 126 typedef struct lsbuf { 127 union { 128 LIST_ENTRY(lsbuf) list; 129 SLIST_ENTRY(lsbuf) slist; 130 TAILQ_ENTRY(lsbuf) tailq; 131 } lb_chain; 132 uintptr_t lb_lock; /* lock address */ 133 uintptr_t lb_callsite; /* call site */ 134 uint64_t lb_times[LB_NEVENT]; /* cumulative times */ 135 uint32_t lb_counts[LB_NEVENT]; /* count of events */ 136 uint16_t lb_flags; /* lock type */ 137 uint16_t lb_cpu; /* CPU number */ 138 } lsbuf_t; 139 140 /* 141 * Tracing stubs used by lock providers. 142 */ 143 144 #if defined(_KERNEL) && defined(__HAVE_CPU_COUNTER) && NLOCKSTAT > 0 145 146 #define LOCKSTAT_EVENT(flag, lock, type, count, time) \ 147 do { \ 148 if (__predict_false(flag)) \ 149 lockstat_event((uintptr_t)(lock), \ 150 (uintptr_t)__builtin_return_address(0), \ 151 (type), (count), (time)); \ 152 } while (/* CONSTCOND */ 0); 153 154 #define LOCKSTAT_EVENT_RA(flag, lock, type, count, time, ra) \ 155 do { \ 156 if (__predict_false(flag)) \ 157 lockstat_event((uintptr_t)(lock), (uintptr_t)ra, \ 158 (type), (count), (time)); \ 159 } while (/* CONSTCOND */ 0); 160 161 #define LOCKSTAT_TIMER(name) uint64_t name = 0 162 #define LOCKSTAT_COUNTER(name) uint64_t name = 0 163 #define LOCKSTAT_FLAG(name) int name 164 #define LOCKSTAT_ENTER(name) name = atomic_load_relaxed(&lockstat_enabled) 165 #define LOCKSTAT_EXIT(name) 166 167 #define LOCKSTAT_START_TIMER(flag, name) \ 168 do { \ 169 if (__predict_false(flag)) \ 170 (name) -= cpu_counter(); \ 171 } while (/* CONSTCOND */ 0) 172 173 #define LOCKSTAT_STOP_TIMER(flag, name) \ 174 do { \ 175 if (__predict_false(flag)) \ 176 (name) += cpu_counter(); \ 177 } while (/* CONSTCOND */ 0) 178 179 #define LOCKSTAT_COUNT(name, inc) \ 180 do { \ 181 (name) += (inc); \ 182 } while (/* CONSTCOND */ 0) 183 184 void lockstat_event(uintptr_t, uintptr_t, u_int, u_int, uint64_t); 185 186 #else 187 188 #define LOCKSTAT_FLAG(name) /* nothing */ 189 #define LOCKSTAT_ENTER(name) /* nothing */ 190 #define LOCKSTAT_EXIT(name) /* nothing */ 191 #define LOCKSTAT_EVENT(flag, lock, type, count, time) /* nothing */ 192 #define LOCKSTAT_EVENT_RA(flag, lock, type, count, time, ra) /* nothing */ 193 #define LOCKSTAT_TIMER(void) /* nothing */ 194 #define LOCKSTAT_COUNTER(void) /* nothing */ 195 #define LOCKSTAT_START_TIMER(flag, void) /* nothing */ 196 #define LOCKSTAT_STOP_TIMER(flag, void) /* nothing */ 197 #define LOCKSTAT_COUNT(name, int) /* nothing */ 198 199 #endif 200 201 #ifdef KDTRACE_HOOKS 202 extern volatile u_int lockstat_dtrace_enabled; 203 #define KDTRACE_LOCKSTAT_ENABLED lockstat_dtrace_enabled 204 #define LS_COMPRESS(f) \ 205 ((((f) & 0x3) | (((f) & 0x700) >> 6)) & (LS_NPROBES - 1)) 206 #define LS_NPROBES 0x20 /* 5 bits */ 207 208 extern uint32_t lockstat_probemap[]; 209 extern void (*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t, 210 uintptr_t, uintptr_t, uintptr_t); 211 212 void lockstat_probe_stub(uint32_t, uintptr_t, uintptr_t, 213 uintptr_t, uintptr_t, uintptr_t); 214 #else 215 #define KDTRACE_LOCKSTAT_ENABLED 0 216 #endif 217 218 #if defined(_KERNEL) && NLOCKSTAT > 0 219 extern __cpu_simple_lock_t lockstat_enabled_lock; 220 extern volatile u_int lockstat_enabled; 221 extern volatile u_int lockstat_dev_enabled; 222 223 #define LOCKSTAT_ENABLED_UPDATE_BEGIN() do \ 224 { \ 225 __cpu_simple_lock(&lockstat_enabled_lock); \ 226 } while (/*CONSTCOND*/0) 227 228 #define LOCKSTAT_ENABLED_UPDATE_END() do \ 229 { \ 230 atomic_store_relaxed(&lockstat_enabled, \ 231 lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED); \ 232 __cpu_simple_unlock(&lockstat_enabled_lock); \ 233 } while (/*CONSTCOND*/0) 234 235 #endif 236 237 #endif /* _SYS_LOCKSTAT_H_ */ 238