1 /* $NetBSD: tprof_armv8.c,v 1.20 2023/04/11 10:07:12 msaitoh Exp $ */
2 
3 /*-
4  * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: tprof_armv8.c,v 1.20 2023/04/11 10:07:12 msaitoh Exp $");
31 
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/cpu.h>
35 #include <sys/percpu.h>
36 #include <sys/xcall.h>
37 
38 #include <dev/tprof/tprof.h>
39 
40 #include <arm/armreg.h>
41 #include <arm/cpufunc.h>
42 
43 #include <dev/tprof/tprof_armv8.h>
44 
45 static u_int counter_bitwidth;
46 
47 /*
48  * armv8 can handle up to 31 event counters,
49  * PMCR_EL0.N counters are actually available.
50  */
51 
52 static bool
armv8_pmu_event_implemented(uint16_t event)53 armv8_pmu_event_implemented(uint16_t event)
54 {
55           uint64_t eid[2];
56 
57           if (event >= 64)
58                     return false;
59 
60           eid[0] = reg_pmceid0_el0_read();
61           eid[1] = reg_pmceid1_el0_read();
62 
63           /* The low 32bits of PMCEID[01]_EL0 contain the common events 0 to n */
64           const u_int idx = event / 32;
65           const u_int bit = event % 32;
66 
67           if (eid[idx] & __BIT(bit))
68                     return true;
69 
70           return false;
71 }
72 
73 static void
armv8_pmu_set_pmevtyper(u_int counter,uint64_t val)74 armv8_pmu_set_pmevtyper(u_int counter, uint64_t val)
75 {
76           reg_pmselr_el0_write(counter);
77           isb();
78           reg_pmxevtyper_el0_write(val);
79 }
80 
81 static inline void
armv8_pmu_set_pmevcntr(u_int counter,uint64_t val)82 armv8_pmu_set_pmevcntr(u_int counter, uint64_t val)
83 {
84           reg_pmselr_el0_write(counter);
85           isb();
86           reg_pmxevcntr_el0_write(val);
87 }
88 
89 static inline uint64_t
armv8_pmu_get_pmevcntr(u_int counter)90 armv8_pmu_get_pmevcntr(u_int counter)
91 {
92           reg_pmselr_el0_write(counter);
93           isb();
94           return reg_pmxevcntr_el0_read();
95 }
96 
97 /* Read and write at once */
98 static inline uint64_t
armv8_pmu_getset_pmevcntr(u_int counter,uint64_t val)99 armv8_pmu_getset_pmevcntr(u_int counter, uint64_t val)
100 {
101           uint64_t c;
102 
103           reg_pmselr_el0_write(counter);
104           isb();
105           c = reg_pmxevcntr_el0_read();
106           reg_pmxevcntr_el0_write(val);
107           return c;
108 }
109 
110 static uint32_t
armv8_pmu_ncounters(void)111 armv8_pmu_ncounters(void)
112 {
113           return __SHIFTOUT(reg_pmcr_el0_read(), PMCR_N);
114 }
115 
116 static u_int
armv8_pmu_counter_bitwidth(u_int counter)117 armv8_pmu_counter_bitwidth(u_int counter)
118 {
119           return counter_bitwidth;
120 }
121 
122 static uint64_t
armv8_pmu_counter_estimate_freq(u_int counter)123 armv8_pmu_counter_estimate_freq(u_int counter)
124 {
125           return curcpu()->ci_data.cpu_cc_freq;
126 }
127 
128 static int
armv8_pmu_valid_event(u_int counter,const tprof_param_t * param)129 armv8_pmu_valid_event(u_int counter, const tprof_param_t *param)
130 {
131           if (!armv8_pmu_event_implemented(param->p_event)) {
132                     printf("%s: event %#" PRIx64 " not implemented on this CPU\n",
133                         __func__, param->p_event);
134                     return EINVAL;
135           }
136           return 0;
137 }
138 
139 static void
armv8_pmu_configure_event(u_int counter,const tprof_param_t * param)140 armv8_pmu_configure_event(u_int counter, const tprof_param_t *param)
141 {
142           /* Disable event counter */
143           reg_pmcntenclr_el0_write(__BIT(counter) & PMCNTEN_P);
144 
145           /* Disable overflow interrupts */
146           reg_pmintenclr_el1_write(__BIT(counter) & PMINTEN_P);
147 
148           /* Configure event counter */
149           uint64_t pmevtyper = __SHIFTIN(param->p_event, PMEVTYPER_EVTCOUNT);
150           if (!ISSET(param->p_flags, TPROF_PARAM_USER))
151                     pmevtyper |= PMEVTYPER_U;
152           if (!ISSET(param->p_flags, TPROF_PARAM_KERN))
153                     pmevtyper |= PMEVTYPER_P;
154           armv8_pmu_set_pmevtyper(counter, pmevtyper);
155 
156           if (ISSET(param->p_flags, TPROF_PARAM_PROFILE) ||
157               counter_bitwidth != 64) {
158                     /* Enable overflow interrupts */
159                     reg_pmintenset_el1_write(__BIT(counter) & PMINTEN_P);
160           }
161 
162           /* Clear overflow flag */
163           reg_pmovsclr_el0_write(__BIT(counter) & PMOVS_P);
164 
165           /* Reset the counter */
166           armv8_pmu_set_pmevcntr(counter, param->p_value);
167 }
168 
169 static void
armv8_pmu_start(tprof_countermask_t runmask)170 armv8_pmu_start(tprof_countermask_t runmask)
171 {
172           /* Enable event counters */
173           reg_pmcntenset_el0_write(runmask & PMCNTEN_P);
174 
175           /*
176            * PMCR.E is shared with PMCCNTR_EL0 and event counters.
177            * It is set here in case PMCCNTR_EL0 is not used in the system.
178            */
179           reg_pmcr_el0_write(reg_pmcr_el0_read() | PMCR_E);
180 }
181 
182 static void
armv8_pmu_stop(tprof_countermask_t stopmask)183 armv8_pmu_stop(tprof_countermask_t stopmask)
184 {
185           /* Disable event counter */
186           reg_pmcntenclr_el0_write(stopmask & PMCNTEN_P);
187 }
188 
189 /* XXX: argument of armv8_pmu_intr() */
190 extern struct tprof_backend *tprof_backend;
191 static void *pmu_intr_arg;
192 
193 int
armv8_pmu_intr(void * priv)194 armv8_pmu_intr(void *priv)
195 {
196           const struct trapframe * const tf = priv;
197           tprof_backend_softc_t *sc = pmu_intr_arg;
198           tprof_frame_info_t tfi;
199           int bit;
200           const uint32_t pmovs = reg_pmovsset_el0_read();
201 
202           uint64_t *counters_offset =
203               percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
204           uint32_t mask = pmovs;
205           while ((bit = ffs(mask)) != 0) {
206                     bit--;
207                     CLR(mask, __BIT(bit));
208 
209                     if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
210                               /* Account for the counter, and reset */
211                               uint64_t ctr = armv8_pmu_getset_pmevcntr(bit,
212                                   sc->sc_count[bit].ctr_counter_reset_val);
213                               counters_offset[bit] +=
214                                   sc->sc_count[bit].ctr_counter_val + ctr;
215 
216                               /* Record a sample */
217                               tfi.tfi_pc = tf->tf_pc;
218                               tfi.tfi_counter = bit;
219                               tfi.tfi_inkernel =
220                                   tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS &&
221                                   tfi.tfi_pc < VM_MAX_KERNEL_ADDRESS;
222                               tprof_sample(NULL, &tfi);
223                     } else if (ISSET(sc->sc_ctr_ovf_mask, __BIT(bit))) {
224                               /* Counter has overflowed */
225                               counters_offset[bit] += __BIT(32);
226                     }
227           }
228           reg_pmovsclr_el0_write(pmovs);
229 
230           return 1;
231 }
232 
233 static uint32_t
armv8_pmu_ident(void)234 armv8_pmu_ident(void)
235 {
236           return TPROF_IDENT_ARMV8_GENERIC;
237 }
238 
239 static const tprof_backend_ops_t tprof_armv8_pmu_ops = {
240           .tbo_ident = armv8_pmu_ident,
241           .tbo_ncounters = armv8_pmu_ncounters,
242           .tbo_counter_bitwidth = armv8_pmu_counter_bitwidth,
243           .tbo_counter_read = armv8_pmu_get_pmevcntr,
244           .tbo_counter_estimate_freq = armv8_pmu_counter_estimate_freq,
245           .tbo_valid_event = armv8_pmu_valid_event,
246           .tbo_configure_event = armv8_pmu_configure_event,
247           .tbo_start = armv8_pmu_start,
248           .tbo_stop = armv8_pmu_stop,
249           .tbo_establish = NULL,
250           .tbo_disestablish = NULL,
251 };
252 
253 static void
armv8_pmu_init_cpu(void * arg1,void * arg2)254 armv8_pmu_init_cpu(void *arg1, void *arg2)
255 {
256           /* Disable EL0 access to performance monitors */
257           reg_pmuserenr_el0_write(0);
258 
259           /* Disable interrupts */
260           reg_pmintenclr_el1_write(PMINTEN_P);
261 
262           /* Disable event counters */
263           reg_pmcntenclr_el0_write(PMCNTEN_P);
264 }
265 
266 bool
armv8_pmu_detect(void)267 armv8_pmu_detect(void)
268 {
269           const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
270           const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
271 
272           return pmuver != ID_AA64DFR0_EL1_PMUVER_NONE &&
273                  pmuver != ID_AA64DFR0_EL1_PMUVER_IMPL;
274 }
275 
276 int
armv8_pmu_init(void)277 armv8_pmu_init(void)
278 {
279           int error, ncounters;
280 
281           KASSERT(armv8_pmu_detect());
282 
283           ncounters = armv8_pmu_ncounters();
284           if (ncounters == 0)
285                     return ENOTSUP;
286 
287           /* Is 64bit event counter available? */
288           const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
289           const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
290           if (pmuver >= ID_AA64DFR0_EL1_PMUVER_V3P5 &&
291               ISSET(reg_pmcr_el0_read(), PMCR_LP))
292                     counter_bitwidth = 64;
293           else
294                     counter_bitwidth = 32;
295 
296           uint64_t xc = xc_broadcast(0, armv8_pmu_init_cpu, NULL, NULL);
297           xc_wait(xc);
298 
299           error = tprof_backend_register("tprof_armv8", &tprof_armv8_pmu_ops,
300               TPROF_BACKEND_VERSION);
301           if (error == 0) {
302                     /* XXX: for argument of armv8_pmu_intr() */
303                     pmu_intr_arg = tprof_backend;
304           }
305 
306           return error;
307 }
308