1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
22 *
23 * $FreeBSD: stable/10/sys/cddl/dev/fbt/fbt.c 282748 2015-05-11 07:54:39Z avg $
24 *
25 */
26
27 /*
28 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
29 * Use is subject to license terms.
30 */
31
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/conf.h>
36 #include <sys/cpuvar.h>
37 #include <sys/fcntl.h>
38 #include <sys/filio.h>
39 #include <sys/kdb.h>
40 #include <sys/kernel.h>
41 #include <sys/kmem.h>
42 #include <sys/kthread.h>
43 #include <sys/limits.h>
44 #include <sys/linker.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/pcpu.h>
50 #include <sys/poll.h>
51 #include <sys/proc.h>
52 #include <sys/selinfo.h>
53 #include <sys/smp.h>
54 #include <sys/syscall.h>
55 #include <sys/sysent.h>
56 #include <sys/sysproto.h>
57 #include <sys/uio.h>
58 #include <sys/unistd.h>
59 #include <machine/stdarg.h>
60
61 #include <sys/dtrace.h>
62 #include <sys/dtrace_bsd.h>
63
64 static MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
65
66 #define FBT_PUSHL_EBP 0x55
67 #define FBT_MOVL_ESP_EBP0_V0 0x8b
68 #define FBT_MOVL_ESP_EBP1_V0 0xec
69 #define FBT_MOVL_ESP_EBP0_V1 0x89
70 #define FBT_MOVL_ESP_EBP1_V1 0xe5
71 #define FBT_REX_RSP_RBP 0x48
72
73 #define FBT_POPL_EBP 0x5d
74 #define FBT_RET 0xc3
75 #define FBT_RET_IMM16 0xc2
76 #define FBT_LEAVE 0xc9
77
78 #ifdef __amd64__
79 #define FBT_PATCHVAL 0xcc
80 #else
81 #define FBT_PATCHVAL 0xf0
82 #endif
83
84 static d_open_t fbt_open;
85 static int fbt_unload(void);
86 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
87 static void fbt_provide_module(void *, modctl_t *);
88 static void fbt_destroy(void *, dtrace_id_t, void *);
89 static void fbt_enable(void *, dtrace_id_t, void *);
90 static void fbt_disable(void *, dtrace_id_t, void *);
91 static void fbt_load(void *);
92 static void fbt_suspend(void *, dtrace_id_t, void *);
93 static void fbt_resume(void *, dtrace_id_t, void *);
94
95 #define FBT_ENTRY "entry"
96 #define FBT_RETURN "return"
97 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
98 #define FBT_PROBETAB_SIZE 0x8000 /* 32k entries -- 128K total */
99
100 static struct cdevsw fbt_cdevsw = {
101 .d_version = D_VERSION,
102 .d_open = fbt_open,
103 .d_name = "fbt",
104 };
105
106 static dtrace_pattr_t fbt_attr = {
107 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
108 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
109 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
110 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
111 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
112 };
113
114 static dtrace_pops_t fbt_pops = {
115 NULL,
116 fbt_provide_module,
117 fbt_enable,
118 fbt_disable,
119 fbt_suspend,
120 fbt_resume,
121 fbt_getargdesc,
122 NULL,
123 NULL,
124 fbt_destroy
125 };
126
127 typedef struct fbt_probe {
128 struct fbt_probe *fbtp_hashnext;
129 uint8_t *fbtp_patchpoint;
130 int8_t fbtp_rval;
131 uint8_t fbtp_patchval;
132 uint8_t fbtp_savedval;
133 uintptr_t fbtp_roffset;
134 dtrace_id_t fbtp_id;
135 const char *fbtp_name;
136 modctl_t *fbtp_ctl;
137 int fbtp_loadcnt;
138 int fbtp_primary;
139 int fbtp_invop_cnt;
140 int fbtp_symindx;
141 struct fbt_probe *fbtp_next;
142 } fbt_probe_t;
143
144 static struct cdev *fbt_cdev;
145 static dtrace_provider_id_t fbt_id;
146 static fbt_probe_t **fbt_probetab;
147 static int fbt_probetab_size;
148 static int fbt_probetab_mask;
149 static int fbt_verbose = 0;
150
151 static void
fbt_doubletrap(void)152 fbt_doubletrap(void)
153 {
154 fbt_probe_t *fbt;
155 int i;
156
157 for (i = 0; i < fbt_probetab_size; i++) {
158 fbt = fbt_probetab[i];
159
160 for (; fbt != NULL; fbt = fbt->fbtp_next)
161 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
162 }
163 }
164
165 static int
fbt_invop(uintptr_t addr,uintptr_t * stack,uintptr_t rval)166 fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
167 {
168 solaris_cpu_t *cpu = &solaris_cpu[curcpu];
169 uintptr_t stack0, stack1, stack2, stack3, stack4;
170 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
171
172 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
173 if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
174 fbt->fbtp_invop_cnt++;
175 if (fbt->fbtp_roffset == 0) {
176 int i = 0;
177 /*
178 * When accessing the arguments on the stack,
179 * we must protect against accessing beyond
180 * the stack. We can safely set NOFAULT here
181 * -- we know that interrupts are already
182 * disabled.
183 */
184 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
185 cpu->cpu_dtrace_caller = stack[i++];
186 stack0 = stack[i++];
187 stack1 = stack[i++];
188 stack2 = stack[i++];
189 stack3 = stack[i++];
190 stack4 = stack[i++];
191 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
192 CPU_DTRACE_BADADDR);
193
194 dtrace_probe(fbt->fbtp_id, stack0, stack1,
195 stack2, stack3, stack4);
196
197 cpu->cpu_dtrace_caller = 0;
198 } else {
199 #ifdef __amd64__
200 /*
201 * On amd64, we instrument the ret, not the
202 * leave. We therefore need to set the caller
203 * to assure that the top frame of a stack()
204 * action is correct.
205 */
206 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
207 cpu->cpu_dtrace_caller = stack[0];
208 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
209 CPU_DTRACE_BADADDR);
210 #endif
211
212 dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
213 rval, 0, 0, 0);
214 cpu->cpu_dtrace_caller = 0;
215 }
216
217 return (fbt->fbtp_rval);
218 }
219 }
220
221 return (0);
222 }
223
224 static int
fbt_provide_module_function(linker_file_t lf,int symindx,linker_symval_t * symval,void * opaque)225 fbt_provide_module_function(linker_file_t lf, int symindx,
226 linker_symval_t *symval, void *opaque)
227 {
228 char *modname = opaque;
229 const char *name = symval->name;
230 fbt_probe_t *fbt, *retfbt;
231 int j;
232 int size;
233 u_int8_t *instr, *limit;
234
235 if (strncmp(name, "dtrace_", 7) == 0 &&
236 strncmp(name, "dtrace_safe_", 12) != 0) {
237 /*
238 * Anything beginning with "dtrace_" may be called
239 * from probe context unless it explicitly indicates
240 * that it won't be called from probe context by
241 * using the prefix "dtrace_safe_".
242 */
243 return (0);
244 }
245
246 if (name[0] == '_' && name[1] == '_')
247 return (0);
248
249 size = symval->size;
250
251 instr = (u_int8_t *) symval->value;
252 limit = (u_int8_t *) symval->value + symval->size;
253
254 #ifdef __amd64__
255 while (instr < limit) {
256 if (*instr == FBT_PUSHL_EBP)
257 break;
258
259 if ((size = dtrace_instr_size(instr)) <= 0)
260 break;
261
262 instr += size;
263 }
264
265 if (instr >= limit || *instr != FBT_PUSHL_EBP) {
266 /*
267 * We either don't save the frame pointer in this
268 * function, or we ran into some disassembly
269 * screw-up. Either way, we bail.
270 */
271 return (0);
272 }
273 #else
274 if (instr[0] != FBT_PUSHL_EBP)
275 return (0);
276
277 if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 &&
278 instr[2] == FBT_MOVL_ESP_EBP1_V0) &&
279 !(instr[1] == FBT_MOVL_ESP_EBP0_V1 &&
280 instr[2] == FBT_MOVL_ESP_EBP1_V1))
281 return (0);
282 #endif
283
284 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
285 fbt->fbtp_name = name;
286 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
287 name, FBT_ENTRY, 3, fbt);
288 fbt->fbtp_patchpoint = instr;
289 fbt->fbtp_ctl = lf;
290 fbt->fbtp_loadcnt = lf->loadcnt;
291 fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP;
292 fbt->fbtp_savedval = *instr;
293 fbt->fbtp_patchval = FBT_PATCHVAL;
294 fbt->fbtp_symindx = symindx;
295
296 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
297 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
298
299 lf->fbt_nentries++;
300
301 retfbt = NULL;
302 again:
303 if (instr >= limit)
304 return (0);
305
306 /*
307 * If this disassembly fails, then we've likely walked off into
308 * a jump table or some other unsuitable area. Bail out of the
309 * disassembly now.
310 */
311 if ((size = dtrace_instr_size(instr)) <= 0)
312 return (0);
313
314 #ifdef __amd64__
315 /*
316 * We only instrument "ret" on amd64 -- we don't yet instrument
317 * ret imm16, largely because the compiler doesn't seem to
318 * (yet) emit them in the kernel...
319 */
320 if (*instr != FBT_RET) {
321 instr += size;
322 goto again;
323 }
324 #else
325 if (!(size == 1 &&
326 (*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) &&
327 (*(instr + 1) == FBT_RET ||
328 *(instr + 1) == FBT_RET_IMM16))) {
329 instr += size;
330 goto again;
331 }
332 #endif
333
334 /*
335 * We (desperately) want to avoid erroneously instrumenting a
336 * jump table, especially given that our markers are pretty
337 * short: two bytes on x86, and just one byte on amd64. To
338 * determine if we're looking at a true instruction sequence
339 * or an inline jump table that happens to contain the same
340 * byte sequences, we resort to some heuristic sleeze: we
341 * treat this instruction as being contained within a pointer,
342 * and see if that pointer points to within the body of the
343 * function. If it does, we refuse to instrument it.
344 */
345 for (j = 0; j < sizeof (uintptr_t); j++) {
346 caddr_t check = (caddr_t) instr - j;
347 uint8_t *ptr;
348
349 if (check < symval->value)
350 break;
351
352 if (check + sizeof (caddr_t) > (caddr_t)limit)
353 continue;
354
355 ptr = *(uint8_t **)check;
356
357 if (ptr >= (uint8_t *) symval->value && ptr < limit) {
358 instr += size;
359 goto again;
360 }
361 }
362
363 /*
364 * We have a winner!
365 */
366 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
367 fbt->fbtp_name = name;
368
369 if (retfbt == NULL) {
370 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
371 name, FBT_RETURN, 3, fbt);
372 } else {
373 retfbt->fbtp_next = fbt;
374 fbt->fbtp_id = retfbt->fbtp_id;
375 }
376
377 retfbt = fbt;
378 fbt->fbtp_patchpoint = instr;
379 fbt->fbtp_ctl = lf;
380 fbt->fbtp_loadcnt = lf->loadcnt;
381 fbt->fbtp_symindx = symindx;
382
383 #ifndef __amd64__
384 if (*instr == FBT_POPL_EBP) {
385 fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
386 } else {
387 ASSERT(*instr == FBT_LEAVE);
388 fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
389 }
390 fbt->fbtp_roffset =
391 (uintptr_t)(instr - (uint8_t *) symval->value) + 1;
392
393 #else
394 ASSERT(*instr == FBT_RET);
395 fbt->fbtp_rval = DTRACE_INVOP_RET;
396 fbt->fbtp_roffset =
397 (uintptr_t)(instr - (uint8_t *) symval->value);
398 #endif
399
400 fbt->fbtp_savedval = *instr;
401 fbt->fbtp_patchval = FBT_PATCHVAL;
402 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
403 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
404
405 lf->fbt_nentries++;
406
407 instr += size;
408 goto again;
409 }
410
411 static void
fbt_provide_module(void * arg,modctl_t * lf)412 fbt_provide_module(void *arg, modctl_t *lf)
413 {
414 char modname[MAXPATHLEN];
415 int i;
416 size_t len;
417
418 strlcpy(modname, lf->filename, sizeof(modname));
419 len = strlen(modname);
420 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0)
421 modname[len - 3] = '\0';
422
423 /*
424 * Employees of dtrace and their families are ineligible. Void
425 * where prohibited.
426 */
427 if (strcmp(modname, "dtrace") == 0)
428 return;
429
430 /*
431 * To register with DTrace, a module must list 'dtrace' as a
432 * dependency in order for the kernel linker to resolve
433 * symbols like dtrace_register(). All modules with such a
434 * dependency are ineligible for FBT tracing.
435 */
436 for (i = 0; i < lf->ndeps; i++)
437 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0)
438 return;
439
440 if (lf->fbt_nentries) {
441 /*
442 * This module has some FBT entries allocated; we're afraid
443 * to screw with it.
444 */
445 return;
446 }
447
448 /*
449 * List the functions in the module and the symbol values.
450 */
451 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname);
452 }
453
454 static void
fbt_destroy(void * arg,dtrace_id_t id,void * parg)455 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
456 {
457 fbt_probe_t *fbt = parg, *next, *hash, *last;
458 modctl_t *ctl;
459 int ndx;
460
461 do {
462 ctl = fbt->fbtp_ctl;
463
464 ctl->fbt_nentries--;
465
466 /*
467 * Now we need to remove this probe from the fbt_probetab.
468 */
469 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
470 last = NULL;
471 hash = fbt_probetab[ndx];
472
473 while (hash != fbt) {
474 ASSERT(hash != NULL);
475 last = hash;
476 hash = hash->fbtp_hashnext;
477 }
478
479 if (last != NULL) {
480 last->fbtp_hashnext = fbt->fbtp_hashnext;
481 } else {
482 fbt_probetab[ndx] = fbt->fbtp_hashnext;
483 }
484
485 next = fbt->fbtp_next;
486 free(fbt, M_FBT);
487
488 fbt = next;
489 } while (fbt != NULL);
490 }
491
492 static void
fbt_enable(void * arg,dtrace_id_t id,void * parg)493 fbt_enable(void *arg, dtrace_id_t id, void *parg)
494 {
495 fbt_probe_t *fbt = parg;
496 modctl_t *ctl = fbt->fbtp_ctl;
497
498 ctl->nenabled++;
499
500 /*
501 * Now check that our modctl has the expected load count. If it
502 * doesn't, this module must have been unloaded and reloaded -- and
503 * we're not going to touch it.
504 */
505 if (ctl->loadcnt != fbt->fbtp_loadcnt) {
506 if (fbt_verbose) {
507 printf("fbt is failing for probe %s "
508 "(module %s reloaded)",
509 fbt->fbtp_name, ctl->filename);
510 }
511
512 return;
513 }
514
515 for (; fbt != NULL; fbt = fbt->fbtp_next) {
516 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
517 }
518 }
519
520 static void
fbt_disable(void * arg,dtrace_id_t id,void * parg)521 fbt_disable(void *arg, dtrace_id_t id, void *parg)
522 {
523 fbt_probe_t *fbt = parg;
524 modctl_t *ctl = fbt->fbtp_ctl;
525
526 ASSERT(ctl->nenabled > 0);
527 ctl->nenabled--;
528
529 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
530 return;
531
532 for (; fbt != NULL; fbt = fbt->fbtp_next)
533 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
534 }
535
536 static void
fbt_suspend(void * arg,dtrace_id_t id,void * parg)537 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
538 {
539 fbt_probe_t *fbt = parg;
540 modctl_t *ctl = fbt->fbtp_ctl;
541
542 ASSERT(ctl->nenabled > 0);
543
544 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
545 return;
546
547 for (; fbt != NULL; fbt = fbt->fbtp_next)
548 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
549 }
550
551 static void
fbt_resume(void * arg,dtrace_id_t id,void * parg)552 fbt_resume(void *arg, dtrace_id_t id, void *parg)
553 {
554 fbt_probe_t *fbt = parg;
555 modctl_t *ctl = fbt->fbtp_ctl;
556
557 ASSERT(ctl->nenabled > 0);
558
559 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
560 return;
561
562 for (; fbt != NULL; fbt = fbt->fbtp_next)
563 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
564 }
565
566 static int
fbt_ctfoff_init(modctl_t * lf,linker_ctf_t * lc)567 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
568 {
569 const Elf_Sym *symp = lc->symtab;;
570 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
571 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
572 int i;
573 uint32_t *ctfoff;
574 uint32_t objtoff = hp->cth_objtoff;
575 uint32_t funcoff = hp->cth_funcoff;
576 ushort_t info;
577 ushort_t vlen;
578
579 /* Sanity check. */
580 if (hp->cth_magic != CTF_MAGIC) {
581 printf("Bad magic value in CTF data of '%s'\n",lf->pathname);
582 return (EINVAL);
583 }
584
585 if (lc->symtab == NULL) {
586 printf("No symbol table in '%s'\n",lf->pathname);
587 return (EINVAL);
588 }
589
590 if ((ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK)) == NULL)
591 return (ENOMEM);
592
593 *lc->ctfoffp = ctfoff;
594
595 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) {
596 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) {
597 *ctfoff = 0xffffffff;
598 continue;
599 }
600
601 switch (ELF_ST_TYPE(symp->st_info)) {
602 case STT_OBJECT:
603 if (objtoff >= hp->cth_funcoff ||
604 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) {
605 *ctfoff = 0xffffffff;
606 break;
607 }
608
609 *ctfoff = objtoff;
610 objtoff += sizeof (ushort_t);
611 break;
612
613 case STT_FUNC:
614 if (funcoff >= hp->cth_typeoff) {
615 *ctfoff = 0xffffffff;
616 break;
617 }
618
619 *ctfoff = funcoff;
620
621 info = *((const ushort_t *)(ctfdata + funcoff));
622 vlen = CTF_INFO_VLEN(info);
623
624 /*
625 * If we encounter a zero pad at the end, just skip it.
626 * Otherwise skip over the function and its return type
627 * (+2) and the argument list (vlen).
628 */
629 if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0)
630 funcoff += sizeof (ushort_t); /* skip pad */
631 else
632 funcoff += sizeof (ushort_t) * (vlen + 2);
633 break;
634
635 default:
636 *ctfoff = 0xffffffff;
637 break;
638 }
639 }
640
641 return (0);
642 }
643
644 static ssize_t
fbt_get_ctt_size(uint8_t version,const ctf_type_t * tp,ssize_t * sizep,ssize_t * incrementp)645 fbt_get_ctt_size(uint8_t version, const ctf_type_t *tp, ssize_t *sizep,
646 ssize_t *incrementp)
647 {
648 ssize_t size, increment;
649
650 if (version > CTF_VERSION_1 &&
651 tp->ctt_size == CTF_LSIZE_SENT) {
652 size = CTF_TYPE_LSIZE(tp);
653 increment = sizeof (ctf_type_t);
654 } else {
655 size = tp->ctt_size;
656 increment = sizeof (ctf_stype_t);
657 }
658
659 if (sizep)
660 *sizep = size;
661 if (incrementp)
662 *incrementp = increment;
663
664 return (size);
665 }
666
667 static int
fbt_typoff_init(linker_ctf_t * lc)668 fbt_typoff_init(linker_ctf_t *lc)
669 {
670 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
671 const ctf_type_t *tbuf;
672 const ctf_type_t *tend;
673 const ctf_type_t *tp;
674 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
675 int ctf_typemax = 0;
676 uint32_t *xp;
677 ulong_t pop[CTF_K_MAX + 1] = { 0 };
678
679
680 /* Sanity check. */
681 if (hp->cth_magic != CTF_MAGIC)
682 return (EINVAL);
683
684 tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff);
685 tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff);
686
687 int child = hp->cth_parname != 0;
688
689 /*
690 * We make two passes through the entire type section. In this first
691 * pass, we count the number of each type and the total number of types.
692 */
693 for (tp = tbuf; tp < tend; ctf_typemax++) {
694 ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
695 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
696 ssize_t size, increment;
697
698 size_t vbytes;
699 uint_t n;
700
701 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
702
703 switch (kind) {
704 case CTF_K_INTEGER:
705 case CTF_K_FLOAT:
706 vbytes = sizeof (uint_t);
707 break;
708 case CTF_K_ARRAY:
709 vbytes = sizeof (ctf_array_t);
710 break;
711 case CTF_K_FUNCTION:
712 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
713 break;
714 case CTF_K_STRUCT:
715 case CTF_K_UNION:
716 if (size < CTF_LSTRUCT_THRESH) {
717 ctf_member_t *mp = (ctf_member_t *)
718 ((uintptr_t)tp + increment);
719
720 vbytes = sizeof (ctf_member_t) * vlen;
721 for (n = vlen; n != 0; n--, mp++)
722 child |= CTF_TYPE_ISCHILD(mp->ctm_type);
723 } else {
724 ctf_lmember_t *lmp = (ctf_lmember_t *)
725 ((uintptr_t)tp + increment);
726
727 vbytes = sizeof (ctf_lmember_t) * vlen;
728 for (n = vlen; n != 0; n--, lmp++)
729 child |=
730 CTF_TYPE_ISCHILD(lmp->ctlm_type);
731 }
732 break;
733 case CTF_K_ENUM:
734 vbytes = sizeof (ctf_enum_t) * vlen;
735 break;
736 case CTF_K_FORWARD:
737 /*
738 * For forward declarations, ctt_type is the CTF_K_*
739 * kind for the tag, so bump that population count too.
740 * If ctt_type is unknown, treat the tag as a struct.
741 */
742 if (tp->ctt_type == CTF_K_UNKNOWN ||
743 tp->ctt_type >= CTF_K_MAX)
744 pop[CTF_K_STRUCT]++;
745 else
746 pop[tp->ctt_type]++;
747 /*FALLTHRU*/
748 case CTF_K_UNKNOWN:
749 vbytes = 0;
750 break;
751 case CTF_K_POINTER:
752 case CTF_K_TYPEDEF:
753 case CTF_K_VOLATILE:
754 case CTF_K_CONST:
755 case CTF_K_RESTRICT:
756 child |= CTF_TYPE_ISCHILD(tp->ctt_type);
757 vbytes = 0;
758 break;
759 default:
760 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
761 return (EIO);
762 }
763 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
764 pop[kind]++;
765 }
766
767 /* account for a sentinel value below */
768 ctf_typemax++;
769 *lc->typlenp = ctf_typemax;
770
771 if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK)) == NULL)
772 return (ENOMEM);
773
774 *lc->typoffp = xp;
775
776 /* type id 0 is used as a sentinel value */
777 *xp++ = 0;
778
779 /*
780 * In the second pass, fill in the type offset.
781 */
782 for (tp = tbuf; tp < tend; xp++) {
783 ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
784 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
785 ssize_t size, increment;
786
787 size_t vbytes;
788 uint_t n;
789
790 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
791
792 switch (kind) {
793 case CTF_K_INTEGER:
794 case CTF_K_FLOAT:
795 vbytes = sizeof (uint_t);
796 break;
797 case CTF_K_ARRAY:
798 vbytes = sizeof (ctf_array_t);
799 break;
800 case CTF_K_FUNCTION:
801 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
802 break;
803 case CTF_K_STRUCT:
804 case CTF_K_UNION:
805 if (size < CTF_LSTRUCT_THRESH) {
806 ctf_member_t *mp = (ctf_member_t *)
807 ((uintptr_t)tp + increment);
808
809 vbytes = sizeof (ctf_member_t) * vlen;
810 for (n = vlen; n != 0; n--, mp++)
811 child |= CTF_TYPE_ISCHILD(mp->ctm_type);
812 } else {
813 ctf_lmember_t *lmp = (ctf_lmember_t *)
814 ((uintptr_t)tp + increment);
815
816 vbytes = sizeof (ctf_lmember_t) * vlen;
817 for (n = vlen; n != 0; n--, lmp++)
818 child |=
819 CTF_TYPE_ISCHILD(lmp->ctlm_type);
820 }
821 break;
822 case CTF_K_ENUM:
823 vbytes = sizeof (ctf_enum_t) * vlen;
824 break;
825 case CTF_K_FORWARD:
826 case CTF_K_UNKNOWN:
827 vbytes = 0;
828 break;
829 case CTF_K_POINTER:
830 case CTF_K_TYPEDEF:
831 case CTF_K_VOLATILE:
832 case CTF_K_CONST:
833 case CTF_K_RESTRICT:
834 vbytes = 0;
835 break;
836 default:
837 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
838 return (EIO);
839 }
840 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata);
841 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
842 }
843
844 return (0);
845 }
846
847 /*
848 * CTF Declaration Stack
849 *
850 * In order to implement ctf_type_name(), we must convert a type graph back
851 * into a C type declaration. Unfortunately, a type graph represents a storage
852 * class ordering of the type whereas a type declaration must obey the C rules
853 * for operator precedence, and the two orderings are frequently in conflict.
854 * For example, consider these CTF type graphs and their C declarations:
855 *
856 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)()
857 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[]
858 *
859 * In each case, parentheses are used to raise operator * to higher lexical
860 * precedence, so the string form of the C declaration cannot be constructed by
861 * walking the type graph links and forming the string from left to right.
862 *
863 * The functions in this file build a set of stacks from the type graph nodes
864 * corresponding to the C operator precedence levels in the appropriate order.
865 * The code in ctf_type_name() can then iterate over the levels and nodes in
866 * lexical precedence order and construct the final C declaration string.
867 */
868 typedef struct ctf_list {
869 struct ctf_list *l_prev; /* previous pointer or tail pointer */
870 struct ctf_list *l_next; /* next pointer or head pointer */
871 } ctf_list_t;
872
873 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev))
874 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next))
875
876 typedef enum {
877 CTF_PREC_BASE,
878 CTF_PREC_POINTER,
879 CTF_PREC_ARRAY,
880 CTF_PREC_FUNCTION,
881 CTF_PREC_MAX
882 } ctf_decl_prec_t;
883
884 typedef struct ctf_decl_node {
885 ctf_list_t cd_list; /* linked list pointers */
886 ctf_id_t cd_type; /* type identifier */
887 uint_t cd_kind; /* type kind */
888 uint_t cd_n; /* type dimension if array */
889 } ctf_decl_node_t;
890
891 typedef struct ctf_decl {
892 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */
893 int cd_order[CTF_PREC_MAX]; /* storage order of decls */
894 ctf_decl_prec_t cd_qualp; /* qualifier precision */
895 ctf_decl_prec_t cd_ordp; /* ordered precision */
896 char *cd_buf; /* buffer for output */
897 char *cd_ptr; /* buffer location */
898 char *cd_end; /* buffer limit */
899 size_t cd_len; /* buffer space required */
900 int cd_err; /* saved error value */
901 } ctf_decl_t;
902
903 /*
904 * Simple doubly-linked list append routine. This implementation assumes that
905 * each list element contains an embedded ctf_list_t as the first member.
906 * An additional ctf_list_t is used to store the head (l_next) and tail
907 * (l_prev) pointers. The current head and tail list elements have their
908 * previous and next pointers set to NULL, respectively.
909 */
910 static void
ctf_list_append(ctf_list_t * lp,void * new)911 ctf_list_append(ctf_list_t *lp, void *new)
912 {
913 ctf_list_t *p = lp->l_prev; /* p = tail list element */
914 ctf_list_t *q = new; /* q = new list element */
915
916 lp->l_prev = q;
917 q->l_prev = p;
918 q->l_next = NULL;
919
920 if (p != NULL)
921 p->l_next = q;
922 else
923 lp->l_next = q;
924 }
925
926 /*
927 * Prepend the specified existing element to the given ctf_list_t. The
928 * existing pointer should be pointing at a struct with embedded ctf_list_t.
929 */
930 static void
ctf_list_prepend(ctf_list_t * lp,void * new)931 ctf_list_prepend(ctf_list_t *lp, void *new)
932 {
933 ctf_list_t *p = new; /* p = new list element */
934 ctf_list_t *q = lp->l_next; /* q = head list element */
935
936 lp->l_next = p;
937 p->l_prev = NULL;
938 p->l_next = q;
939
940 if (q != NULL)
941 q->l_prev = p;
942 else
943 lp->l_prev = p;
944 }
945
946 static void
ctf_decl_init(ctf_decl_t * cd,char * buf,size_t len)947 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len)
948 {
949 int i;
950
951 bzero(cd, sizeof (ctf_decl_t));
952
953 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++)
954 cd->cd_order[i] = CTF_PREC_BASE - 1;
955
956 cd->cd_qualp = CTF_PREC_BASE;
957 cd->cd_ordp = CTF_PREC_BASE;
958
959 cd->cd_buf = buf;
960 cd->cd_ptr = buf;
961 cd->cd_end = buf + len;
962 }
963
964 static void
ctf_decl_fini(ctf_decl_t * cd)965 ctf_decl_fini(ctf_decl_t *cd)
966 {
967 ctf_decl_node_t *cdp, *ndp;
968 int i;
969
970 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) {
971 for (cdp = ctf_list_next(&cd->cd_nodes[i]);
972 cdp != NULL; cdp = ndp) {
973 ndp = ctf_list_next(cdp);
974 free(cdp, M_FBT);
975 }
976 }
977 }
978
979 static const ctf_type_t *
ctf_lookup_by_id(linker_ctf_t * lc,ctf_id_t type)980 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type)
981 {
982 const ctf_type_t *tp;
983 uint32_t offset;
984 uint32_t *typoff = *lc->typoffp;
985
986 if (type >= *lc->typlenp) {
987 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp);
988 return(NULL);
989 }
990
991 /* Check if the type isn't cross-referenced. */
992 if ((offset = typoff[type]) == 0) {
993 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type);
994 return(NULL);
995 }
996
997 tp = (const ctf_type_t *)(lc->ctftab + offset + sizeof(ctf_header_t));
998
999 return (tp);
1000 }
1001
1002 static void
fbt_array_info(linker_ctf_t * lc,ctf_id_t type,ctf_arinfo_t * arp)1003 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp)
1004 {
1005 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
1006 const ctf_type_t *tp;
1007 const ctf_array_t *ap;
1008 ssize_t increment;
1009
1010 bzero(arp, sizeof(*arp));
1011
1012 if ((tp = ctf_lookup_by_id(lc, type)) == NULL)
1013 return;
1014
1015 if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY)
1016 return;
1017
1018 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment);
1019
1020 ap = (const ctf_array_t *)((uintptr_t)tp + increment);
1021 arp->ctr_contents = ap->cta_contents;
1022 arp->ctr_index = ap->cta_index;
1023 arp->ctr_nelems = ap->cta_nelems;
1024 }
1025
1026 static const char *
ctf_strptr(linker_ctf_t * lc,int name)1027 ctf_strptr(linker_ctf_t *lc, int name)
1028 {
1029 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;;
1030 const char *strp = "";
1031
1032 if (name < 0 || name >= hp->cth_strlen)
1033 return(strp);
1034
1035 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t));
1036
1037 return (strp);
1038 }
1039
1040 static void
ctf_decl_push(ctf_decl_t * cd,linker_ctf_t * lc,ctf_id_t type)1041 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type)
1042 {
1043 ctf_decl_node_t *cdp;
1044 ctf_decl_prec_t prec;
1045 uint_t kind, n = 1;
1046 int is_qual = 0;
1047
1048 const ctf_type_t *tp;
1049 ctf_arinfo_t ar;
1050
1051 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) {
1052 cd->cd_err = ENOENT;
1053 return;
1054 }
1055
1056 switch (kind = CTF_INFO_KIND(tp->ctt_info)) {
1057 case CTF_K_ARRAY:
1058 fbt_array_info(lc, type, &ar);
1059 ctf_decl_push(cd, lc, ar.ctr_contents);
1060 n = ar.ctr_nelems;
1061 prec = CTF_PREC_ARRAY;
1062 break;
1063
1064 case CTF_K_TYPEDEF:
1065 if (ctf_strptr(lc, tp->ctt_name)[0] == '\0') {
1066 ctf_decl_push(cd, lc, tp->ctt_type);
1067 return;
1068 }
1069 prec = CTF_PREC_BASE;
1070 break;
1071
1072 case CTF_K_FUNCTION:
1073 ctf_decl_push(cd, lc, tp->ctt_type);
1074 prec = CTF_PREC_FUNCTION;
1075 break;
1076
1077 case CTF_K_POINTER:
1078 ctf_decl_push(cd, lc, tp->ctt_type);
1079 prec = CTF_PREC_POINTER;
1080 break;
1081
1082 case CTF_K_VOLATILE:
1083 case CTF_K_CONST:
1084 case CTF_K_RESTRICT:
1085 ctf_decl_push(cd, lc, tp->ctt_type);
1086 prec = cd->cd_qualp;
1087 is_qual++;
1088 break;
1089
1090 default:
1091 prec = CTF_PREC_BASE;
1092 }
1093
1094 if ((cdp = malloc(sizeof (ctf_decl_node_t), M_FBT, M_WAITOK)) == NULL) {
1095 cd->cd_err = EAGAIN;
1096 return;
1097 }
1098
1099 cdp->cd_type = type;
1100 cdp->cd_kind = kind;
1101 cdp->cd_n = n;
1102
1103 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL)
1104 cd->cd_order[prec] = cd->cd_ordp++;
1105
1106 /*
1107 * Reset cd_qualp to the highest precedence level that we've seen so
1108 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER).
1109 */
1110 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY)
1111 cd->cd_qualp = prec;
1112
1113 /*
1114 * C array declarators are ordered inside out so prepend them. Also by
1115 * convention qualifiers of base types precede the type specifier (e.g.
1116 * const int vs. int const) even though the two forms are equivalent.
1117 */
1118 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE))
1119 ctf_list_prepend(&cd->cd_nodes[prec], cdp);
1120 else
1121 ctf_list_append(&cd->cd_nodes[prec], cdp);
1122 }
1123
1124 static void
ctf_decl_sprintf(ctf_decl_t * cd,const char * format,...)1125 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...)
1126 {
1127 size_t len = (size_t)(cd->cd_end - cd->cd_ptr);
1128 va_list ap;
1129 size_t n;
1130
1131 va_start(ap, format);
1132 n = vsnprintf(cd->cd_ptr, len, format, ap);
1133 va_end(ap);
1134
1135 cd->cd_ptr += MIN(n, len);
1136 cd->cd_len += n;
1137 }
1138
1139 static ssize_t
fbt_type_name(linker_ctf_t * lc,ctf_id_t type,char * buf,size_t len)1140 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len)
1141 {
1142 ctf_decl_t cd;
1143 ctf_decl_node_t *cdp;
1144 ctf_decl_prec_t prec, lp, rp;
1145 int ptr, arr;
1146 uint_t k;
1147
1148 if (lc == NULL && type == CTF_ERR)
1149 return (-1); /* simplify caller code by permitting CTF_ERR */
1150
1151 ctf_decl_init(&cd, buf, len);
1152 ctf_decl_push(&cd, lc, type);
1153
1154 if (cd.cd_err != 0) {
1155 ctf_decl_fini(&cd);
1156 return (-1);
1157 }
1158
1159 /*
1160 * If the type graph's order conflicts with lexical precedence order
1161 * for pointers or arrays, then we need to surround the declarations at
1162 * the corresponding lexical precedence with parentheses. This can
1163 * result in either a parenthesized pointer (*) as in int (*)() or
1164 * int (*)[], or in a parenthesized pointer and array as in int (*[])().
1165 */
1166 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER;
1167 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY;
1168
1169 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1;
1170 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1;
1171
1172 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */
1173
1174 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) {
1175 for (cdp = ctf_list_next(&cd.cd_nodes[prec]);
1176 cdp != NULL; cdp = ctf_list_next(cdp)) {
1177
1178 const ctf_type_t *tp =
1179 ctf_lookup_by_id(lc, cdp->cd_type);
1180 const char *name = ctf_strptr(lc, tp->ctt_name);
1181
1182 if (k != CTF_K_POINTER && k != CTF_K_ARRAY)
1183 ctf_decl_sprintf(&cd, " ");
1184
1185 if (lp == prec) {
1186 ctf_decl_sprintf(&cd, "(");
1187 lp = -1;
1188 }
1189
1190 switch (cdp->cd_kind) {
1191 case CTF_K_INTEGER:
1192 case CTF_K_FLOAT:
1193 case CTF_K_TYPEDEF:
1194 ctf_decl_sprintf(&cd, "%s", name);
1195 break;
1196 case CTF_K_POINTER:
1197 ctf_decl_sprintf(&cd, "*");
1198 break;
1199 case CTF_K_ARRAY:
1200 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n);
1201 break;
1202 case CTF_K_FUNCTION:
1203 ctf_decl_sprintf(&cd, "()");
1204 break;
1205 case CTF_K_STRUCT:
1206 case CTF_K_FORWARD:
1207 ctf_decl_sprintf(&cd, "struct %s", name);
1208 break;
1209 case CTF_K_UNION:
1210 ctf_decl_sprintf(&cd, "union %s", name);
1211 break;
1212 case CTF_K_ENUM:
1213 ctf_decl_sprintf(&cd, "enum %s", name);
1214 break;
1215 case CTF_K_VOLATILE:
1216 ctf_decl_sprintf(&cd, "volatile");
1217 break;
1218 case CTF_K_CONST:
1219 ctf_decl_sprintf(&cd, "const");
1220 break;
1221 case CTF_K_RESTRICT:
1222 ctf_decl_sprintf(&cd, "restrict");
1223 break;
1224 }
1225
1226 k = cdp->cd_kind;
1227 }
1228
1229 if (rp == prec)
1230 ctf_decl_sprintf(&cd, ")");
1231 }
1232
1233 ctf_decl_fini(&cd);
1234 return (cd.cd_len);
1235 }
1236
1237 static void
fbt_getargdesc(void * arg __unused,dtrace_id_t id __unused,void * parg,dtrace_argdesc_t * desc)1238 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc)
1239 {
1240 const ushort_t *dp;
1241 fbt_probe_t *fbt = parg;
1242 linker_ctf_t lc;
1243 modctl_t *ctl = fbt->fbtp_ctl;
1244 int ndx = desc->dtargd_ndx;
1245 int symindx = fbt->fbtp_symindx;
1246 uint32_t *ctfoff;
1247 uint32_t offset;
1248 ushort_t info, kind, n;
1249
1250 if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) {
1251 (void) strcpy(desc->dtargd_native, "int");
1252 return;
1253 }
1254
1255 desc->dtargd_ndx = DTRACE_ARGNONE;
1256
1257 /* Get a pointer to the CTF data and it's length. */
1258 if (linker_ctf_get(ctl, &lc) != 0)
1259 /* No CTF data? Something wrong? *shrug* */
1260 return;
1261
1262 /* Check if this module hasn't been initialised yet. */
1263 if (*lc.ctfoffp == NULL) {
1264 /*
1265 * Initialise the CTF object and function symindx to
1266 * byte offset array.
1267 */
1268 if (fbt_ctfoff_init(ctl, &lc) != 0)
1269 return;
1270
1271 /* Initialise the CTF type to byte offset array. */
1272 if (fbt_typoff_init(&lc) != 0)
1273 return;
1274 }
1275
1276 ctfoff = *lc.ctfoffp;
1277
1278 if (ctfoff == NULL || *lc.typoffp == NULL)
1279 return;
1280
1281 /* Check if the symbol index is out of range. */
1282 if (symindx >= lc.nsym)
1283 return;
1284
1285 /* Check if the symbol isn't cross-referenced. */
1286 if ((offset = ctfoff[symindx]) == 0xffffffff)
1287 return;
1288
1289 dp = (const ushort_t *)(lc.ctftab + offset + sizeof(ctf_header_t));
1290
1291 info = *dp++;
1292 kind = CTF_INFO_KIND(info);
1293 n = CTF_INFO_VLEN(info);
1294
1295 if (kind == CTF_K_UNKNOWN && n == 0) {
1296 printf("%s(%d): Unknown function!\n",__func__,__LINE__);
1297 return;
1298 }
1299
1300 if (kind != CTF_K_FUNCTION) {
1301 printf("%s(%d): Expected a function!\n",__func__,__LINE__);
1302 return;
1303 }
1304
1305 if (fbt->fbtp_roffset != 0) {
1306 /* Only return type is available for args[1] in return probe. */
1307 if (ndx > 1)
1308 return;
1309 ASSERT(ndx == 1);
1310 } else {
1311 /* Check if the requested argument doesn't exist. */
1312 if (ndx >= n)
1313 return;
1314
1315 /* Skip the return type and arguments up to the one requested. */
1316 dp += ndx + 1;
1317 }
1318
1319 if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0)
1320 desc->dtargd_ndx = ndx;
1321
1322 return;
1323 }
1324
1325 static int
fbt_linker_file_cb(linker_file_t lf,void * arg)1326 fbt_linker_file_cb(linker_file_t lf, void *arg)
1327 {
1328
1329 fbt_provide_module(arg, lf);
1330
1331 return (0);
1332 }
1333
1334 static void
fbt_load(void * dummy)1335 fbt_load(void *dummy)
1336 {
1337 /* Create the /dev/dtrace/fbt entry. */
1338 fbt_cdev = make_dev(&fbt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
1339 "dtrace/fbt");
1340
1341 /* Default the probe table size if not specified. */
1342 if (fbt_probetab_size == 0)
1343 fbt_probetab_size = FBT_PROBETAB_SIZE;
1344
1345 /* Choose the hash mask for the probe table. */
1346 fbt_probetab_mask = fbt_probetab_size - 1;
1347
1348 /* Allocate memory for the probe table. */
1349 fbt_probetab =
1350 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO);
1351
1352 dtrace_doubletrap_func = fbt_doubletrap;
1353 dtrace_invop_add(fbt_invop);
1354
1355 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
1356 NULL, &fbt_pops, NULL, &fbt_id) != 0)
1357 return;
1358
1359 /* Create probes for the kernel and already-loaded modules. */
1360 linker_file_foreach(fbt_linker_file_cb, NULL);
1361 }
1362
1363 static int
fbt_unload()1364 fbt_unload()
1365 {
1366 int error = 0;
1367
1368 /* De-register the invalid opcode handler. */
1369 dtrace_invop_remove(fbt_invop);
1370
1371 dtrace_doubletrap_func = NULL;
1372
1373 /* De-register this DTrace provider. */
1374 if ((error = dtrace_unregister(fbt_id)) != 0)
1375 return (error);
1376
1377 /* Free the probe table. */
1378 free(fbt_probetab, M_FBT);
1379 fbt_probetab = NULL;
1380 fbt_probetab_mask = 0;
1381
1382 destroy_dev(fbt_cdev);
1383
1384 return (error);
1385 }
1386
1387 static int
fbt_modevent(module_t mod __unused,int type,void * data __unused)1388 fbt_modevent(module_t mod __unused, int type, void *data __unused)
1389 {
1390 int error = 0;
1391
1392 switch (type) {
1393 case MOD_LOAD:
1394 break;
1395
1396 case MOD_UNLOAD:
1397 break;
1398
1399 case MOD_SHUTDOWN:
1400 break;
1401
1402 default:
1403 error = EOPNOTSUPP;
1404 break;
1405
1406 }
1407
1408 return (error);
1409 }
1410
1411 static int
fbt_open(struct cdev * dev __unused,int oflags __unused,int devtype __unused,struct thread * td __unused)1412 fbt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
1413 {
1414 return (0);
1415 }
1416
1417 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL);
1418 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL);
1419
1420 DEV_MODULE(fbt, fbt_modevent, NULL);
1421 MODULE_VERSION(fbt, 1);
1422 MODULE_DEPEND(fbt, dtrace, 1, 1, 1);
1423 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1);
1424