1 /* $NetBSD: tprof_x86_intel.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $ */
2
3 /*
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Maxime Villard.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c)2008,2009 YAMAMOTO Takashi,
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 *
45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * SUCH DAMAGE.
56 */
57
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_intel.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $");
60
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/kernel.h>
64 #include <sys/module.h>
65
66 #include <sys/cpu.h>
67 #include <sys/percpu.h>
68 #include <sys/xcall.h>
69
70 #include <dev/tprof/tprof.h>
71
72 #include <uvm/uvm.h> /* VM_MIN_KERNEL_ADDRESS */
73
74 #include <x86/nmi.h>
75
76 #include <machine/cpufunc.h>
77 #include <machine/cputypes.h> /* CPUVENDOR_* */
78 #include <machine/cpuvar.h> /* cpu_vendor */
79 #include <machine/i82489reg.h>
80 #include <machine/i82489var.h>
81
82 static u_int counter_bitwidth;
83
84 #define PERFEVTSEL(i) (MSR_EVNTSEL0 + (i))
85 #define PERFCTR(i) (MSR_PERFCTR0 + (i))
86
87 #define PERFEVTSEL_EVENT_SELECT __BITS(0, 7)
88 #define PERFEVTSEL_UNIT_MASK __BITS(8, 15)
89 #define PERFEVTSEL_USR __BIT(16)
90 #define PERFEVTSEL_OS __BIT(17)
91 #define PERFEVTSEL_E __BIT(18)
92 #define PERFEVTSEL_PC __BIT(19)
93 #define PERFEVTSEL_INT __BIT(20)
94 #define PERFEVTSEL_EN __BIT(22)
95 #define PERFEVTSEL_INV __BIT(23)
96 #define PERFEVTSEL_COUNTER_MASK __BITS(24, 31)
97
98 static uint32_t intel_lapic_saved[MAXCPUS];
99 static nmi_handler_t *intel_nmi_handle;
100
101 static uint32_t
tprof_intel_ncounters(void)102 tprof_intel_ncounters(void)
103 {
104 uint32_t descs[4];
105
106 if (cpuid_level < 0x0a)
107 return 0;
108
109 x86_cpuid(0x0a, descs);
110
111 return __SHIFTOUT(descs[0], CPUID_PERF_NGPPC);
112 }
113
114 static u_int
tprof_intel_counter_bitwidth(u_int counter)115 tprof_intel_counter_bitwidth(u_int counter)
116 {
117
118 return counter_bitwidth;
119 }
120
121 static inline void
tprof_intel_counter_write(u_int counter,uint64_t val)122 tprof_intel_counter_write(u_int counter, uint64_t val)
123 {
124
125 wrmsr(PERFCTR(counter), val);
126 }
127
128 static inline uint64_t
tprof_intel_counter_read(u_int counter)129 tprof_intel_counter_read(u_int counter)
130 {
131
132 return rdmsr(PERFCTR(counter));
133 }
134
135 static void
tprof_intel_configure_event(u_int counter,const tprof_param_t * param)136 tprof_intel_configure_event(u_int counter, const tprof_param_t *param)
137 {
138 uint64_t evtval;
139
140 evtval =
141 __SHIFTIN(param->p_event, PERFEVTSEL_EVENT_SELECT) |
142 __SHIFTIN(param->p_unit, PERFEVTSEL_UNIT_MASK) |
143 ((param->p_flags & TPROF_PARAM_USER) ? PERFEVTSEL_USR : 0) |
144 ((param->p_flags & TPROF_PARAM_KERN) ? PERFEVTSEL_OS : 0) |
145 PERFEVTSEL_INT;
146 wrmsr(PERFEVTSEL(counter), evtval);
147
148 /* Reset the counter */
149 tprof_intel_counter_write(counter, param->p_value);
150 }
151
152 static void
tprof_intel_start(tprof_countermask_t runmask)153 tprof_intel_start(tprof_countermask_t runmask)
154 {
155 int bit;
156
157 while ((bit = ffs(runmask)) != 0) {
158 bit--;
159 CLR(runmask, __BIT(bit));
160 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PERFEVTSEL_EN);
161 }
162 }
163
164 static void
tprof_intel_stop(tprof_countermask_t stopmask)165 tprof_intel_stop(tprof_countermask_t stopmask)
166 {
167 int bit;
168
169 while ((bit = ffs(stopmask)) != 0) {
170 bit--;
171 CLR(stopmask, __BIT(bit));
172 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) &
173 ~PERFEVTSEL_EN);
174 }
175 }
176
177 static int
tprof_intel_nmi(const struct trapframe * tf,void * arg)178 tprof_intel_nmi(const struct trapframe *tf, void *arg)
179 {
180 tprof_backend_softc_t *sc = arg;
181 tprof_frame_info_t tfi;
182 uint32_t pcint;
183 int bit;
184
185 uint64_t *counters_offset =
186 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
187 tprof_countermask_t mask = sc->sc_ctr_ovf_mask;
188 while ((bit = ffs(mask)) != 0) {
189 bit--;
190 CLR(mask, __BIT(bit));
191
192 /* If the highest bit is non zero, then it's not for us. */
193 uint64_t ctr = tprof_intel_counter_read(bit);
194 if ((ctr & __BIT(counter_bitwidth - 1)) != 0)
195 continue; /* not overflowed */
196
197 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
198 /* Account for the counter, and reset */
199 tprof_intel_counter_write(bit,
200 sc->sc_count[bit].ctr_counter_reset_val);
201 counters_offset[bit] +=
202 sc->sc_count[bit].ctr_counter_val + ctr;
203
204 /* Record a sample */
205 #if defined(__x86_64__)
206 tfi.tfi_pc = tf->tf_rip;
207 #else
208 tfi.tfi_pc = tf->tf_eip;
209 #endif
210 tfi.tfi_counter = bit;
211 tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS;
212 tprof_sample(NULL, &tfi);
213 } else {
214 /* Not profiled, but require to consider overflow */
215 counters_offset[bit] += __BIT(counter_bitwidth);
216 }
217 }
218
219 /* Unmask PMI */
220 pcint = lapic_readreg(LAPIC_LVT_PCINT);
221 KASSERT((pcint & LAPIC_LVT_MASKED) != 0);
222 lapic_writereg(LAPIC_LVT_PCINT, pcint & ~LAPIC_LVT_MASKED);
223
224 return 1;
225 }
226
227 static uint64_t
tprof_intel_counter_estimate_freq(u_int counter)228 tprof_intel_counter_estimate_freq(u_int counter)
229 {
230
231 return curcpu()->ci_data.cpu_cc_freq;
232 }
233
234 static uint32_t
tprof_intel_ident(void)235 tprof_intel_ident(void)
236 {
237 uint32_t descs[4];
238
239 if (cpu_vendor != CPUVENDOR_INTEL)
240 return TPROF_IDENT_NONE;
241
242 if (cpuid_level < 0x0a)
243 return TPROF_IDENT_NONE;
244
245 x86_cpuid(0x0a, descs);
246 if ((descs[0] & CPUID_PERF_VERSION) == 0)
247 return TPROF_IDENT_NONE;
248
249 if ((descs[0] & CPUID_PERF_NGPPC) == 0)
250 return TPROF_IDENT_NONE;
251
252 counter_bitwidth = __SHIFTOUT(descs[0], CPUID_PERF_NBWGPPC);
253
254 return TPROF_IDENT_INTEL_GENERIC;
255 }
256
257 static void
tprof_intel_establish_cpu(void * arg1,void * arg2)258 tprof_intel_establish_cpu(void *arg1, void *arg2)
259 {
260 struct cpu_info * const ci = curcpu();
261
262 intel_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT);
263 lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI);
264 }
265
266 static void
tprof_intel_disestablish_cpu(void * arg1,void * arg2)267 tprof_intel_disestablish_cpu(void *arg1, void *arg2)
268 {
269 struct cpu_info * const ci = curcpu();
270
271 lapic_writereg(LAPIC_LVT_PCINT, intel_lapic_saved[cpu_index(ci)]);
272 }
273
274 static int
tprof_intel_establish(tprof_backend_softc_t * sc)275 tprof_intel_establish(tprof_backend_softc_t *sc)
276 {
277 uint64_t xc;
278
279 if (tprof_intel_ident() == TPROF_IDENT_NONE)
280 return ENOTSUP;
281
282 KASSERT(intel_nmi_handle == NULL);
283 intel_nmi_handle = nmi_establish(tprof_intel_nmi, sc);
284
285 xc = xc_broadcast(0, tprof_intel_establish_cpu, sc, NULL);
286 xc_wait(xc);
287
288 return 0;
289 }
290
291 static void
tprof_intel_disestablish(tprof_backend_softc_t * sc)292 tprof_intel_disestablish(tprof_backend_softc_t *sc)
293 {
294 uint64_t xc;
295
296 xc = xc_broadcast(0, tprof_intel_disestablish_cpu, sc, NULL);
297 xc_wait(xc);
298
299 KASSERT(intel_nmi_handle != NULL);
300 nmi_disestablish(intel_nmi_handle);
301 intel_nmi_handle = NULL;
302 }
303
304 const tprof_backend_ops_t tprof_intel_ops = {
305 .tbo_ident = tprof_intel_ident,
306 .tbo_ncounters = tprof_intel_ncounters,
307 .tbo_counter_bitwidth = tprof_intel_counter_bitwidth,
308 .tbo_counter_read = tprof_intel_counter_read,
309 .tbo_counter_estimate_freq = tprof_intel_counter_estimate_freq,
310 .tbo_valid_event = NULL,
311 .tbo_configure_event = tprof_intel_configure_event,
312 .tbo_start = tprof_intel_start,
313 .tbo_stop = tprof_intel_stop,
314 .tbo_establish = tprof_intel_establish,
315 .tbo_disestablish = tprof_intel_disestablish,
316 };
317