463 #define	P4_CPU_TO_FLAG(C)		(P4_CPU_IS_HTT_SECONDARY(cpu) ? 0x2 : 0x1)
523 #define	P4_CPU_IS_HTT_SECONDARY(cpu)					\  argument
524 	(p4_system_has_htt ? ((cpu) & 1) : 0)
525 #define	P4_TO_HTT_PRIMARY(cpu) 						\  argument
526 	(p4_system_has_htt ? ((cpu) & ~1) : (cpu))
555 p4_pcpu_init(struct pmc_mdep *md, int cpu)  in p4_pcpu_init()  argument
563 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_pcpu_init()
564 	    ("[p4,%d] insane cpu number %d", __LINE__, cpu));  in p4_pcpu_init()
566 	PMCDBG2(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,  in p4_pcpu_init()
567 	    pmc_cpu_is_primary(cpu) != 0);  in p4_pcpu_init()
583 	if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) {  in p4_pcpu_init()
587 		phycpu = P4_TO_HTT_PRIMARY(cpu);  in p4_pcpu_init()
589 		plc = pmc_pcpu[cpu];  in p4_pcpu_init()
593 		PMCDBG3(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,  in p4_pcpu_init()
596 		    __LINE__, cpu, phycpu));  in p4_pcpu_init()
611 	pc = pmc_pcpu[cpu];  in p4_pcpu_init()
613 	KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));  in p4_pcpu_init()
615 	p4_pcpu[cpu] = p4c;  in p4_pcpu_init()
620 		    PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);  in p4_pcpu_init()
639 p4_pcpu_fini(struct pmc_mdep *md, int cpu)  in p4_pcpu_fini()  argument
645 	PMCDBG1(MDP,INI,0, "p4-cleanup cpu=%d", cpu);  in p4_pcpu_fini()
647 	pc = pmc_pcpu[cpu];  in p4_pcpu_fini()
653 	if (!pmc_cpu_is_primary(cpu) && (cpu & 1))  in p4_pcpu_fini()
656 	p4c = p4_pcpu[cpu];  in p4_pcpu_fini()
669 	p4_pcpu[cpu] = NULL;  in p4_pcpu_fini()
679 p4_read_pmc(int cpu, int ri, pmc_value_t *v)  in p4_read_pmc()  argument
687 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_read_pmc()
688 	    ("[p4,%d] illegal CPU value %d", __LINE__, cpu));  in p4_read_pmc()
692 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_read_pmc()
697 	    ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));  in p4_read_pmc()
705 	PMCDBG3(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);  in p4_read_pmc()
713 		if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */  in p4_read_pmc()
715 			    P4_PCPU_HW_VALUE(pc,ri,cpu);  in p4_read_pmc()
717 			tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);  in p4_read_pmc()
718 		tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);  in p4_read_pmc()
736 p4_write_pmc(int cpu, int ri, pmc_value_t v)  in p4_write_pmc()  argument
744 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_write_pmc()
745 	    ("[amd,%d] illegal CPU value %d", __LINE__, cpu));  in p4_write_pmc()
749 	pc  = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_write_pmc()
756 		cpu, ri));  in p4_write_pmc()
760 	PMCDBG4(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,  in p4_write_pmc()
774 		P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;  in p4_write_pmc()
791 p4_config_pmc(int cpu, int ri, struct pmc *pm)  in p4_config_pmc()  argument
797 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_config_pmc()
798 	    ("[p4,%d] illegal CPU %d", __LINE__, cpu));  in p4_config_pmc()
803 	PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);  in p4_config_pmc()
805 	pc  = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_config_pmc()
817 		cfgflags, cpu, ri));  in p4_config_pmc()
821 		__LINE__, cpu, ri));  in p4_config_pmc()
823 	cpuflag = P4_CPU_TO_FLAG(cpu);  in p4_config_pmc()
831 			__LINE__, cpu, ri, pm, phw->phw_pmc));  in p4_config_pmc()
843 		cfgflags, cpu, ri));  in p4_config_pmc()
857 p4_get_config(int cpu, int ri, struct pmc **ppm)  in p4_get_config()  argument
862 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_get_config()
863 	    ("[p4,%d] illegal CPU %d", __LINE__, cpu));  in p4_get_config()
867 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_get_config()
873 	if (cfgflags & P4_CPU_TO_FLAG(cpu))  in p4_get_config()
916 p4_allocate_pmc(int cpu, int ri, struct pmc *pm,  in p4_allocate_pmc()  argument
926 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_allocate_pmc()
927 	    ("[p4,%d] illegal CPU %d", __LINE__, cpu));  in p4_allocate_pmc()
983 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_allocate_pmc()
1120 p4_release_pmc(int cpu, int ri, struct pmc *pm)  in p4_release_pmc()  argument
1130 	PMCDBG3(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);  in p4_release_pmc()
1133 		pc  = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_release_pmc()
1154 p4_start_pmc(int cpu, int ri)  in p4_start_pmc()  argument
1162 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_start_pmc()
1163 	    ("[p4,%d] illegal CPU value %d", __LINE__, cpu));  in p4_start_pmc()
1167 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_start_pmc()
1172 	    ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));  in p4_start_pmc()
1174 	PMCDBG2(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri);  in p4_start_pmc()
1191 	if (P4_CPU_IS_HTT_SECONDARY(cpu)) { /* shift T0 bits to T1 position */  in p4_start_pmc()
1217 	    ("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,  in p4_start_pmc()
1224 			cpu, ri, pd->pm_cccr_msr));  in p4_start_pmc()
1228 		    P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);  in p4_start_pmc()
1245 			"cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,  in p4_start_pmc()
1250 			__LINE__, rc, cpu, ri));  in p4_start_pmc()
1253 		     "cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,  in p4_start_pmc()
1261 			"escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,  in p4_start_pmc()
1267 			rc, cpu, ri, escrmsr, escrvalue, escrtbits));  in p4_start_pmc()
1275 	P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);  in p4_start_pmc()
1287 	    "escrmsr=0x%x escrvalue=0x%x", cpu, rc,  in p4_start_pmc()
1290 	    cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));  in p4_start_pmc()
1300 p4_stop_pmc(int cpu, int ri)  in p4_stop_pmc()  argument
1309 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_stop_pmc()
1310 	    ("[p4,%d] illegal CPU value %d", __LINE__, cpu));  in p4_stop_pmc()
1315 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_stop_pmc()
1319 	    ("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));  in p4_stop_pmc()
1321 	PMCDBG2(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);  in p4_stop_pmc()
1343 	if (P4_CPU_IS_HTT_SECONDARY(cpu)) {  in p4_stop_pmc()
1353 	    ("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,  in p4_stop_pmc()
1370 		"escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,  in p4_stop_pmc()
1375 		"tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));  in p4_stop_pmc()
1390 	    "escrval=0x%x", cpu, rc, ri, escrmsr, escrvalue);  in p4_stop_pmc()
1393 	if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */  in p4_stop_pmc()
1394 		tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);  in p4_stop_pmc()
1396 		tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);  in p4_stop_pmc()
1398 	P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;  in p4_stop_pmc()
1418 p4_intr(int cpu, struct trapframe *tf)  in p4_intr()  argument
1426 	PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,  in p4_intr()
1429 	pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];  in p4_intr()
1431 	ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ?  in p4_intr()
1435 		ovf_partner = P4_CPU_IS_HTT_SECONDARY(cpu) ?  in p4_intr()
1470 			(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,  in p4_intr()
1520 		error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,  in p4_intr()
1560 p4_describe(int cpu, int ri, struct pmc_info *pi,  in p4_describe()  argument
1567 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in p4_describe()
1568 	    ("[p4,%d] illegal CPU %d", __LINE__, cpu));  in p4_describe()
1572 	PMCDBG2(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri);  in p4_describe()
1574 	if (P4_CPU_IS_HTT_SECONDARY(cpu))  in p4_describe()
1585 	if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {  in p4_describe()
1587 		*ppmc          = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;  in p4_describe()