1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 *
22 * $FreeBSD$
23 *
24 */
25 /*
26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/types.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/kmem.h>
39 #include <sys/smp.h>
40 #include <sys/dtrace_impl.h>
41 #include <sys/dtrace_bsd.h>
42 #include <machine/clock.h>
43 #include <machine/frame.h>
44 #include <machine/trap.h>
45 #include <vm/pmap.h>
46
47 #define DELAYBRANCH(x) ((int)(x) < 0)
48
49 extern uintptr_t dtrace_in_probe_addr;
50 extern int dtrace_in_probe;
51 extern dtrace_id_t dtrace_probeid_error;
52 extern int (*dtrace_invop_jump_addr)(struct trapframe *);
53
54 extern void dtrace_getnanotime(struct timespec *tsp);
55
56 int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t);
57 void dtrace_invop_init(void);
58 void dtrace_invop_uninit(void);
59
60 typedef struct dtrace_invop_hdlr {
61 int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t);
62 struct dtrace_invop_hdlr *dtih_next;
63 } dtrace_invop_hdlr_t;
64
65 dtrace_invop_hdlr_t *dtrace_invop_hdlr;
66
67 int
dtrace_invop(uintptr_t addr,uintptr_t * stack,uintptr_t arg0)68 dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t arg0)
69 {
70 dtrace_invop_hdlr_t *hdlr;
71 int rval;
72
73 for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
74 if ((rval = hdlr->dtih_func(addr, stack, arg0)) != 0)
75 return (rval);
76
77 return (0);
78 }
79
80 void
dtrace_invop_add(int (* func)(uintptr_t,uintptr_t *,uintptr_t))81 dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
82 {
83 dtrace_invop_hdlr_t *hdlr;
84
85 hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
86 hdlr->dtih_func = func;
87 hdlr->dtih_next = dtrace_invop_hdlr;
88 dtrace_invop_hdlr = hdlr;
89 }
90
91 void
dtrace_invop_remove(int (* func)(uintptr_t,uintptr_t *,uintptr_t))92 dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
93 {
94 dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL;
95
96 for (;;) {
97 if (hdlr == NULL)
98 panic("attempt to remove non-existent invop handler");
99
100 if (hdlr->dtih_func == func)
101 break;
102
103 prev = hdlr;
104 hdlr = hdlr->dtih_next;
105 }
106
107 if (prev == NULL) {
108 ASSERT(dtrace_invop_hdlr == hdlr);
109 dtrace_invop_hdlr = hdlr->dtih_next;
110 } else {
111 ASSERT(dtrace_invop_hdlr != hdlr);
112 prev->dtih_next = hdlr->dtih_next;
113 }
114
115 kmem_free(hdlr, 0);
116 }
117
118
119 /*ARGSUSED*/
120 void
dtrace_toxic_ranges(void (* func)(uintptr_t base,uintptr_t limit))121 dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
122 {
123 /*
124 * No toxic regions?
125 */
126 }
127
128 void
dtrace_xcall(processorid_t cpu,dtrace_xcall_t func,void * arg)129 dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
130 {
131 cpuset_t cpus;
132
133 if (cpu == DTRACE_CPUALL)
134 cpus = all_cpus;
135 else
136 CPU_SETOF(cpu, &cpus);
137
138 smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func,
139 smp_no_rendevous_barrier, arg);
140 }
141
142 static void
dtrace_sync_func(void)143 dtrace_sync_func(void)
144 {
145 }
146
147 void
dtrace_sync(void)148 dtrace_sync(void)
149 {
150 dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
151 }
152
153 static int64_t tgt_cpu_tsc;
154 static int64_t hst_cpu_tsc;
155 static int64_t timebase_skew[MAXCPU];
156 static uint64_t nsec_scale;
157
158 /* See below for the explanation of this macro. */
159 /* This is taken from the amd64 dtrace_subr, to provide a synchronized timer
160 * between multiple processors in dtrace. Since PowerPC Timebases can be much
161 * lower than x86, the scale shift is 26 instead of 28, allowing for a 15.63MHz
162 * timebase.
163 */
164 #define SCALE_SHIFT 26
165
166 static void
dtrace_gethrtime_init_cpu(void * arg)167 dtrace_gethrtime_init_cpu(void *arg)
168 {
169 uintptr_t cpu = (uintptr_t) arg;
170
171 if (cpu == curcpu)
172 tgt_cpu_tsc = mftb();
173 else
174 hst_cpu_tsc = mftb();
175 }
176
177 static void
dtrace_gethrtime_init(void * arg)178 dtrace_gethrtime_init(void *arg)
179 {
180 struct pcpu *pc;
181 uint64_t tb_f;
182 cpuset_t map;
183 int i;
184
185 tb_f = cpu_tickrate();
186
187 /*
188 * The following line checks that nsec_scale calculated below
189 * doesn't overflow 32-bit unsigned integer, so that it can multiply
190 * another 32-bit integer without overflowing 64-bit.
191 * Thus minimum supported Timebase frequency is 15.63MHz.
192 */
193 KASSERT(tb_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("Timebase frequency is too low"));
194
195 /*
196 * We scale up NANOSEC/tb_f ratio to preserve as much precision
197 * as possible.
198 * 2^26 factor was chosen quite arbitrarily from practical
199 * considerations:
200 * - it supports TSC frequencies as low as 15.63MHz (see above);
201 */
202 nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tb_f;
203
204 /* The current CPU is the reference one. */
205 sched_pin();
206 timebase_skew[curcpu] = 0;
207 CPU_FOREACH(i) {
208 if (i == curcpu)
209 continue;
210
211 pc = pcpu_find(i);
212 CPU_SETOF(PCPU_GET(cpuid), &map);
213 CPU_SET(pc->pc_cpuid, &map);
214
215 smp_rendezvous_cpus(map, NULL,
216 dtrace_gethrtime_init_cpu,
217 smp_no_rendevous_barrier, (void *)(uintptr_t) i);
218
219 timebase_skew[i] = tgt_cpu_tsc - hst_cpu_tsc;
220 }
221 sched_unpin();
222 }
223
224 SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
225
226 /*
227 * DTrace needs a high resolution time function which can
228 * be called from a probe context and guaranteed not to have
229 * instrumented with probes itself.
230 *
231 * Returns nanoseconds since boot.
232 */
233 uint64_t
dtrace_gethrtime()234 dtrace_gethrtime()
235 {
236 uint64_t timebase;
237 uint32_t lo;
238 uint32_t hi;
239
240 /*
241 * We split timebase value into lower and higher 32-bit halves and separately
242 * scale them with nsec_scale, then we scale them down by 2^28
243 * (see nsec_scale calculations) taking into account 32-bit shift of
244 * the higher half and finally add.
245 */
246 timebase = mftb() - timebase_skew[curcpu];
247 lo = timebase;
248 hi = timebase >> 32;
249 return (((lo * nsec_scale) >> SCALE_SHIFT) +
250 ((hi * nsec_scale) << (32 - SCALE_SHIFT)));
251 }
252
253 uint64_t
dtrace_gethrestime(void)254 dtrace_gethrestime(void)
255 {
256 struct timespec curtime;
257
258 dtrace_getnanotime(&curtime);
259
260 return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec);
261 }
262
263 /* Function to handle DTrace traps during probes. See powerpc/powerpc/trap.c */
264 int
dtrace_trap(struct trapframe * frame,u_int type)265 dtrace_trap(struct trapframe *frame, u_int type)
266 {
267 /*
268 * A trap can occur while DTrace executes a probe. Before
269 * executing the probe, DTrace blocks re-scheduling and sets
270 * a flag in it's per-cpu flags to indicate that it doesn't
271 * want to fault. On returning from the probe, the no-fault
272 * flag is cleared and finally re-scheduling is enabled.
273 *
274 * Check if DTrace has enabled 'no-fault' mode:
275 *
276 */
277 if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) {
278 /*
279 * There are only a couple of trap types that are expected.
280 * All the rest will be handled in the usual way.
281 */
282 switch (type) {
283 /* Page fault. */
284 case EXC_DSI:
285 case EXC_DSE:
286 /* Flag a bad address. */
287 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
288 cpu_core[curcpu].cpuc_dtrace_illval = frame->cpu.aim.dar;
289
290 /*
291 * Offset the instruction pointer to the instruction
292 * following the one causing the fault.
293 */
294 frame->srr0 += sizeof(int);
295 return (1);
296 case EXC_ISI:
297 case EXC_ISE:
298 /* Flag a bad address. */
299 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
300 cpu_core[curcpu].cpuc_dtrace_illval = frame->srr0;
301
302 /*
303 * Offset the instruction pointer to the instruction
304 * following the one causing the fault.
305 */
306 frame->srr0 += sizeof(int);
307 return (1);
308 default:
309 /* Handle all other traps in the usual way. */
310 break;
311 }
312 }
313
314 /* Handle the trap in the usual way. */
315 return (0);
316 }
317
318 void
dtrace_probe_error(dtrace_state_t * state,dtrace_epid_t epid,int which,int fault,int fltoffs,uintptr_t illval)319 dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
320 int fault, int fltoffs, uintptr_t illval)
321 {
322
323 dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state,
324 (uintptr_t)epid,
325 (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs);
326 }
327
328 static int
dtrace_invop_start(struct trapframe * frame)329 dtrace_invop_start(struct trapframe *frame)
330 {
331 switch (dtrace_invop(frame->srr0, (uintptr_t *)frame, frame->fixreg[3])) {
332 case DTRACE_INVOP_JUMP:
333 break;
334 case DTRACE_INVOP_BCTR:
335 frame->srr0 = frame->ctr;
336 break;
337 case DTRACE_INVOP_BLR:
338 frame->srr0 = frame->lr;
339 break;
340 case DTRACE_INVOP_MFLR_R0:
341 frame->fixreg[0] = frame->lr;
342 frame->srr0 = frame->srr0 + 4;
343 break;
344 default:
345 return (-1);
346 break;
347 }
348
349 return (0);
350 }
351
dtrace_invop_init(void)352 void dtrace_invop_init(void)
353 {
354 dtrace_invop_jump_addr = dtrace_invop_start;
355 }
356
dtrace_invop_uninit(void)357 void dtrace_invop_uninit(void)
358 {
359 dtrace_invop_jump_addr = 0;
360 }
361