1 /*-
2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by the University of Cambridge Computer
6 * Laboratory with support from ARM Ltd.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/pmc.h>
34 #include <sys/pmckern.h>
35
36 #include <machine/pmc_mdep.h>
37 #include <machine/cpu.h>
38
39 static int arm64_npmcs;
40
41 struct arm64_event_code_map {
42 enum pmc_event pe_ev;
43 uint8_t pe_code;
44 };
45
46 /*
47 * Per-processor information.
48 */
49 struct arm64_cpu {
50 struct pmc_hw *pc_arm64pmcs;
51 };
52
53 static struct arm64_cpu **arm64_pcpu;
54
55 /*
56 * Interrupt Enable Set Register
57 */
58 static __inline void
arm64_interrupt_enable(uint32_t pmc)59 arm64_interrupt_enable(uint32_t pmc)
60 {
61 uint32_t reg;
62
63 reg = (1 << pmc);
64 WRITE_SPECIALREG(pmintenset_el1, reg);
65
66 isb();
67 }
68
69 /*
70 * Interrupt Clear Set Register
71 */
72 static __inline void
arm64_interrupt_disable(uint32_t pmc)73 arm64_interrupt_disable(uint32_t pmc)
74 {
75 uint32_t reg;
76
77 reg = (1 << pmc);
78 WRITE_SPECIALREG(pmintenclr_el1, reg);
79
80 isb();
81 }
82
83 /*
84 * Counter Set Enable Register
85 */
86 static __inline void
arm64_counter_enable(unsigned int pmc)87 arm64_counter_enable(unsigned int pmc)
88 {
89 uint32_t reg;
90
91 reg = (1 << pmc);
92 WRITE_SPECIALREG(pmcntenset_el0, reg);
93
94 isb();
95 }
96
97 /*
98 * Counter Clear Enable Register
99 */
100 static __inline void
arm64_counter_disable(unsigned int pmc)101 arm64_counter_disable(unsigned int pmc)
102 {
103 uint32_t reg;
104
105 reg = (1 << pmc);
106 WRITE_SPECIALREG(pmcntenclr_el0, reg);
107
108 isb();
109 }
110
111 /*
112 * Performance Monitors Control Register
113 */
114 static uint32_t
arm64_pmcr_read(void)115 arm64_pmcr_read(void)
116 {
117 uint32_t reg;
118
119 reg = READ_SPECIALREG(pmcr_el0);
120
121 return (reg);
122 }
123
124 static void
arm64_pmcr_write(uint32_t reg)125 arm64_pmcr_write(uint32_t reg)
126 {
127
128 WRITE_SPECIALREG(pmcr_el0, reg);
129
130 isb();
131 }
132
133 /*
134 * Performance Count Register N
135 */
136 static uint32_t
arm64_pmcn_read(unsigned int pmc)137 arm64_pmcn_read(unsigned int pmc)
138 {
139
140 KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
141
142 WRITE_SPECIALREG(pmselr_el0, pmc);
143
144 isb();
145
146 return (READ_SPECIALREG(pmxevcntr_el0));
147 }
148
149 static void
arm64_pmcn_write(unsigned int pmc,uint32_t reg)150 arm64_pmcn_write(unsigned int pmc, uint32_t reg)
151 {
152
153 KASSERT(pmc < arm64_npmcs, ("%s: illegal PMC number %d", __func__, pmc));
154
155 WRITE_SPECIALREG(pmselr_el0, pmc);
156 WRITE_SPECIALREG(pmxevcntr_el0, reg);
157
158 isb();
159 }
160
161 static int
arm64_allocate_pmc(int cpu,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)162 arm64_allocate_pmc(int cpu, int ri, struct pmc *pm,
163 const struct pmc_op_pmcallocate *a)
164 {
165 uint32_t config;
166 struct arm64_cpu *pac;
167 enum pmc_event pe;
168
169 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
170 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
171 KASSERT(ri >= 0 && ri < arm64_npmcs,
172 ("[arm64,%d] illegal row index %d", __LINE__, ri));
173
174 pac = arm64_pcpu[cpu];
175
176 if (a->pm_class != PMC_CLASS_ARMV8) {
177 return (EINVAL);
178 }
179 pe = a->pm_ev;
180
181 /* Adjust the config value if needed. */
182 config = a->pm_md.pm_md_config;
183 if ((a->pm_md.pm_md_flags & PM_MD_RAW_EVENT) == 0) {
184 config = (uint32_t)pe - PMC_EV_ARMV8_FIRST;
185 if (config > (PMC_EV_ARMV8_LAST - PMC_EV_ARMV8_FIRST))
186 return (EINVAL);
187 }
188
189 switch (a->pm_caps & (PMC_CAP_SYSTEM | PMC_CAP_USER)) {
190 case PMC_CAP_SYSTEM:
191 config |= PMEVTYPER_U;
192 break;
193 case PMC_CAP_USER:
194 config |= PMEVTYPER_P;
195 break;
196 default:
197 /*
198 * Trace both USER and SYSTEM if none are specified
199 * (default setting) or if both flags are specified
200 * (user explicitly requested both qualifiers).
201 */
202 break;
203 }
204
205 pm->pm_md.pm_arm64.pm_arm64_evsel = config;
206 PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config);
207
208 return (0);
209 }
210
211
212 static int
arm64_read_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t * v)213 arm64_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
214 {
215 pmc_value_t tmp;
216 register_t s;
217 int reg;
218
219 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
220 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
221 KASSERT(ri >= 0 && ri < arm64_npmcs,
222 ("[arm64,%d] illegal row index %d", __LINE__, ri));
223
224 /*
225 * Ensure we don't get interrupted while updating the overflow count.
226 */
227 s = intr_disable();
228 tmp = arm64_pmcn_read(ri);
229 reg = (1 << ri);
230 if ((READ_SPECIALREG(pmovsclr_el0) & reg) != 0) {
231 /* Clear Overflow Flag */
232 WRITE_SPECIALREG(pmovsclr_el0, reg);
233 pm->pm_pcpu_state[cpu].pps_overflowcnt++;
234
235 /* Reread counter in case we raced. */
236 tmp = arm64_pmcn_read(ri);
237 }
238 tmp += 0x100000000llu * pm->pm_pcpu_state[cpu].pps_overflowcnt;
239 intr_restore(s);
240
241 PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp);
242 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
243 /*
244 * Clamp value to 0 if the counter just overflowed,
245 * otherwise the returned reload count would wrap to a
246 * huge value.
247 */
248 if ((tmp & (1ull << 63)) == 0)
249 tmp = 0;
250 else
251 tmp = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
252 }
253 *v = tmp;
254
255 return (0);
256 }
257
258 static int
arm64_write_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t v)259 arm64_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
260 {
261
262 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
263 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
264 KASSERT(ri >= 0 && ri < arm64_npmcs,
265 ("[arm64,%d] illegal row-index %d", __LINE__, ri));
266
267 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
268 v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
269
270 PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v);
271
272 pm->pm_pcpu_state[cpu].pps_overflowcnt = v >> 32;
273 arm64_pmcn_write(ri, v);
274
275 return (0);
276 }
277
278 static int
arm64_config_pmc(int cpu,int ri,struct pmc * pm)279 arm64_config_pmc(int cpu, int ri, struct pmc *pm)
280 {
281 struct pmc_hw *phw;
282
283 PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
284
285 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
286 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
287 KASSERT(ri >= 0 && ri < arm64_npmcs,
288 ("[arm64,%d] illegal row-index %d", __LINE__, ri));
289
290 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
291
292 KASSERT(pm == NULL || phw->phw_pmc == NULL,
293 ("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
294 __LINE__, pm, phw->phw_pmc));
295
296 phw->phw_pmc = pm;
297
298 return (0);
299 }
300
301 static int
arm64_start_pmc(int cpu,int ri,struct pmc * pm)302 arm64_start_pmc(int cpu, int ri, struct pmc *pm)
303 {
304 uint32_t config;
305
306 config = pm->pm_md.pm_arm64.pm_arm64_evsel;
307
308 /*
309 * Configure the event selection.
310 */
311 WRITE_SPECIALREG(pmselr_el0, ri);
312 WRITE_SPECIALREG(pmxevtyper_el0, config);
313
314 isb();
315
316 /*
317 * Enable the PMC.
318 */
319 arm64_interrupt_enable(ri);
320 arm64_counter_enable(ri);
321
322 return (0);
323 }
324
325 static int
arm64_stop_pmc(int cpu,int ri,struct pmc * pm __unused)326 arm64_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
327 {
328
329 /*
330 * Disable the PMCs.
331 */
332 arm64_counter_disable(ri);
333 arm64_interrupt_disable(ri);
334
335 return (0);
336 }
337
338 static int
arm64_release_pmc(int cpu,int ri,struct pmc * pmc)339 arm64_release_pmc(int cpu, int ri, struct pmc *pmc)
340 {
341 struct pmc_hw *phw;
342
343 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
344 ("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
345 KASSERT(ri >= 0 && ri < arm64_npmcs,
346 ("[arm64,%d] illegal row-index %d", __LINE__, ri));
347
348 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
349 KASSERT(phw->phw_pmc == NULL,
350 ("[arm64,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
351
352 return (0);
353 }
354
355 static int
arm64_intr(struct trapframe * tf)356 arm64_intr(struct trapframe *tf)
357 {
358 struct arm64_cpu *pc;
359 int retval, ri;
360 struct pmc *pm;
361 int error;
362 int reg, cpu;
363
364 cpu = curcpu;
365 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
366 ("[arm64,%d] CPU %d out of range", __LINE__, cpu));
367
368 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *)tf,
369 TRAPF_USERMODE(tf));
370
371 retval = 0;
372 pc = arm64_pcpu[cpu];
373
374 for (ri = 0; ri < arm64_npmcs; ri++) {
375 pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
376 if (pm == NULL)
377 continue;
378 /* Check if counter is overflowed */
379 reg = (1 << ri);
380 if ((READ_SPECIALREG(pmovsclr_el0) & reg) == 0)
381 continue;
382 /* Clear Overflow Flag */
383 WRITE_SPECIALREG(pmovsclr_el0, reg);
384
385 isb();
386
387 retval = 1; /* Found an interrupting PMC. */
388
389 pm->pm_pcpu_state[cpu].pps_overflowcnt += 1;
390
391 if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
392 continue;
393
394 if (pm->pm_state != PMC_STATE_RUNNING)
395 continue;
396
397 error = pmc_process_interrupt(PMC_HR, pm, tf);
398 if (error)
399 arm64_stop_pmc(cpu, ri, pm);
400
401 /* Reload sampling count */
402 arm64_write_pmc(cpu, ri, pm, pm->pm_sc.pm_reloadcount);
403 }
404
405 return (retval);
406 }
407
408 static int
arm64_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)409 arm64_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
410 {
411 struct pmc_hw *phw;
412
413 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
414 ("[arm64,%d], illegal CPU %d", __LINE__, cpu));
415 KASSERT(ri >= 0 && ri < arm64_npmcs,
416 ("[arm64,%d] row-index %d out of range", __LINE__, ri));
417
418 phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
419
420 snprintf(pi->pm_name, sizeof(pi->pm_name), "ARMV8-%d", ri);
421 pi->pm_class = PMC_CLASS_ARMV8;
422
423 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
424 pi->pm_enabled = TRUE;
425 *ppmc = phw->phw_pmc;
426 } else {
427 pi->pm_enabled = FALSE;
428 *ppmc = NULL;
429 }
430
431 return (0);
432 }
433
434 static int
arm64_get_config(int cpu,int ri,struct pmc ** ppm)435 arm64_get_config(int cpu, int ri, struct pmc **ppm)
436 {
437
438 *ppm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
439
440 return (0);
441 }
442
443 static int
arm64_pcpu_init(struct pmc_mdep * md,int cpu)444 arm64_pcpu_init(struct pmc_mdep *md, int cpu)
445 {
446 struct arm64_cpu *pac;
447 struct pmc_hw *phw;
448 struct pmc_cpu *pc;
449 uint64_t pmcr;
450 int first_ri;
451 int i;
452
453 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
454 ("[arm64,%d] wrong cpu number %d", __LINE__, cpu));
455 PMCDBG0(MDP, INI, 1, "arm64-pcpu-init");
456
457 arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC,
458 M_WAITOK | M_ZERO);
459
460 pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs,
461 M_PMC, M_WAITOK | M_ZERO);
462 pc = pmc_pcpu[cpu];
463 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri;
464 KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__));
465
466 for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) {
467 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
468 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
469 phw->phw_pmc = NULL;
470 pc->pc_hwpmcs[i + first_ri] = phw;
471 }
472
473 /*
474 * Disable all counters and overflow interrupts. Upon reset they are in
475 * an undefined state.
476 *
477 * Don't issue an isb here, just wait for the one in arm64_pmcr_write()
478 * to make the writes visible.
479 */
480 WRITE_SPECIALREG(pmcntenclr_el0, 0xffffffff);
481 WRITE_SPECIALREG(pmintenclr_el1, 0xffffffff);
482
483 /* Enable unit */
484 pmcr = arm64_pmcr_read();
485 pmcr |= PMCR_E;
486 arm64_pmcr_write(pmcr);
487
488 return (0);
489 }
490
491 static int
arm64_pcpu_fini(struct pmc_mdep * md,int cpu)492 arm64_pcpu_fini(struct pmc_mdep *md, int cpu)
493 {
494 uint32_t pmcr;
495
496 PMCDBG0(MDP, INI, 1, "arm64-pcpu-fini");
497
498 pmcr = arm64_pmcr_read();
499 pmcr &= ~PMCR_E;
500 arm64_pmcr_write(pmcr);
501
502 free(arm64_pcpu[cpu]->pc_arm64pmcs, M_PMC);
503 free(arm64_pcpu[cpu], M_PMC);
504 arm64_pcpu[cpu] = NULL;
505
506 return (0);
507 }
508
509 struct pmc_mdep *
pmc_arm64_initialize(void)510 pmc_arm64_initialize(void)
511 {
512 struct pmc_mdep *pmc_mdep;
513 struct pmc_classdep *pcd;
514 int idcode, impcode;
515 int reg;
516 uint64_t midr;
517
518 reg = arm64_pmcr_read();
519 arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT;
520 impcode = (reg & PMCR_IMP_MASK) >> PMCR_IMP_SHIFT;
521 idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT;
522
523 PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs);
524
525 /*
526 * Write the CPU model to kern.hwpmc.cpuid.
527 *
528 * We zero the variant and revision fields.
529 *
530 * TODO: how to handle differences between cores due to big.LITTLE?
531 * For now, just use MIDR from CPU 0.
532 */
533 midr = (uint64_t)(pcpu_find(0)->pc_midr);
534 midr &= ~(CPU_VAR_MASK | CPU_REV_MASK);
535 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "0x%016lx", midr);
536
537 /*
538 * Allocate space for pointers to PMC HW descriptors and for
539 * the MDEP structure used by MI code.
540 */
541 arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(),
542 M_PMC, M_WAITOK | M_ZERO);
543
544 /* Just one class */
545 pmc_mdep = pmc_mdep_alloc(1);
546
547 switch(impcode) {
548 case PMCR_IMP_ARM:
549 switch (idcode) {
550 case PMCR_IDCODE_CORTEX_A76:
551 case PMCR_IDCODE_NEOVERSE_N1:
552 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A76;
553 break;
554 case PMCR_IDCODE_CORTEX_A57:
555 case PMCR_IDCODE_CORTEX_A72:
556 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57;
557 break;
558 default:
559 case PMCR_IDCODE_CORTEX_A53:
560 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
561 break;
562 }
563 break;
564 default:
565 pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
566 break;
567 }
568
569 pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8];
570 pcd->pcd_caps = ARMV8_PMC_CAPS;
571 pcd->pcd_class = PMC_CLASS_ARMV8;
572 pcd->pcd_num = arm64_npmcs;
573 pcd->pcd_ri = pmc_mdep->pmd_npmc;
574 pcd->pcd_width = 32;
575
576 pcd->pcd_allocate_pmc = arm64_allocate_pmc;
577 pcd->pcd_config_pmc = arm64_config_pmc;
578 pcd->pcd_pcpu_fini = arm64_pcpu_fini;
579 pcd->pcd_pcpu_init = arm64_pcpu_init;
580 pcd->pcd_describe = arm64_describe;
581 pcd->pcd_get_config = arm64_get_config;
582 pcd->pcd_read_pmc = arm64_read_pmc;
583 pcd->pcd_release_pmc = arm64_release_pmc;
584 pcd->pcd_start_pmc = arm64_start_pmc;
585 pcd->pcd_stop_pmc = arm64_stop_pmc;
586 pcd->pcd_write_pmc = arm64_write_pmc;
587
588 pmc_mdep->pmd_intr = arm64_intr;
589 pmc_mdep->pmd_npmc += arm64_npmcs;
590
591 return (pmc_mdep);
592 }
593
594 void
pmc_arm64_finalize(struct pmc_mdep * md)595 pmc_arm64_finalize(struct pmc_mdep *md)
596 {
597 PMCDBG0(MDP, INI, 1, "arm64-finalize");
598
599 free(arm64_pcpu, M_PMC);
600 }
601