1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h 313176 2017-02-03 22:26:19Z gnn $ 22 */ 23 24 /* 25 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 /* 30 * Copyright 2016 Joyent, Inc. 31 * Copyright (c) 2012 by Delphix. All rights reserved. 32 */ 33 34 #ifndef _SYS_DTRACE_IMPL_H 35 #define _SYS_DTRACE_IMPL_H 36 37 #ifdef __cplusplus 38 extern "C" { 39 #endif 40 41 /* 42 * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces 43 * 44 * Note: The contents of this file are private to the implementation of the 45 * Solaris system and DTrace subsystem and are subject to change at any time 46 * without notice. Applications and drivers using these interfaces will fail 47 * to run on future releases. These interfaces should not be used for any 48 * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB). 49 * Please refer to the "Solaris Dynamic Tracing Guide" for more information. 50 */ 51 52 #include <sys/dtrace.h> 53 54 #ifndef illumos 55 #ifdef __sparcv9 56 typedef uint32_t pc_t; 57 #else 58 typedef uintptr_t pc_t; 59 #endif 60 typedef u_long greg_t; 61 #endif 62 63 /* 64 * DTrace Implementation Constants and Typedefs 65 */ 66 #define DTRACE_MAXPROPLEN 128 67 #define DTRACE_DYNVAR_CHUNKSIZE 256 68 69 #ifdef __FreeBSD__ 70 #define NCPU MAXCPU 71 #endif /* __FreeBSD__ */ 72 #ifdef __NetBSD__ 73 #define NCPU MAXCPUS 74 #endif /* __NetBSD__ */ 75 76 struct dtrace_probe; 77 struct dtrace_ecb; 78 struct dtrace_predicate; 79 struct dtrace_action; 80 struct dtrace_provider; 81 struct dtrace_state; 82 83 typedef struct dtrace_probe dtrace_probe_t; 84 typedef struct dtrace_ecb dtrace_ecb_t; 85 typedef struct dtrace_predicate dtrace_predicate_t; 86 typedef struct dtrace_action dtrace_action_t; 87 typedef struct dtrace_provider dtrace_provider_t; 88 typedef struct dtrace_meta dtrace_meta_t; 89 typedef struct dtrace_state dtrace_state_t; 90 typedef uint32_t dtrace_optid_t; 91 typedef uint32_t dtrace_specid_t; 92 typedef uint64_t dtrace_genid_t; 93 94 /* 95 * DTrace Probes 96 * 97 * The probe is the fundamental unit of the DTrace architecture. Probes are 98 * created by DTrace providers, and managed by the DTrace framework. A probe 99 * is identified by a unique <provider, module, function, name> tuple, and has 100 * a unique probe identifier assigned to it. (Some probes are not associated 101 * with a specific point in text; these are called _unanchored probes_ and have 102 * no module or function associated with them.) Probes are represented as a 103 * dtrace_probe structure. To allow quick lookups based on each element of the 104 * probe tuple, probes are hashed by each of provider, module, function and 105 * name. (If a lookup is performed based on a regular expression, a 106 * dtrace_probekey is prepared, and a linear search is performed.) Each probe 107 * is additionally pointed to by a linear array indexed by its identifier. The 108 * identifier is the provider's mechanism for indicating to the DTrace 109 * framework that a probe has fired: the identifier is passed as the first 110 * argument to dtrace_probe(), where it is then mapped into the corresponding 111 * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can 112 * iterate over the probe's list of enabling control blocks; see "DTrace 113 * Enabling Control Blocks", below.) 114 */ 115 struct dtrace_probe { 116 dtrace_id_t dtpr_id; /* probe identifier */ 117 dtrace_ecb_t *dtpr_ecb; /* ECB list; see below */ 118 dtrace_ecb_t *dtpr_ecb_last; /* last ECB in list */ 119 void *dtpr_arg; /* provider argument */ 120 dtrace_cacheid_t dtpr_predcache; /* predicate cache ID */ 121 int dtpr_aframes; /* artificial frames */ 122 dtrace_provider_t *dtpr_provider; /* pointer to provider */ 123 char *dtpr_mod; /* probe's module name */ 124 char *dtpr_func; /* probe's function name */ 125 char *dtpr_name; /* probe's name */ 126 dtrace_probe_t *dtpr_nextmod; /* next in module hash */ 127 dtrace_probe_t *dtpr_prevmod; /* previous in module hash */ 128 dtrace_probe_t *dtpr_nextfunc; /* next in function hash */ 129 dtrace_probe_t *dtpr_prevfunc; /* previous in function hash */ 130 dtrace_probe_t *dtpr_nextname; /* next in name hash */ 131 dtrace_probe_t *dtpr_prevname; /* previous in name hash */ 132 dtrace_genid_t dtpr_gen; /* probe generation ID */ 133 }; 134 135 typedef int dtrace_probekey_f(const char *, const char *, int); 136 137 typedef struct dtrace_probekey { 138 char *dtpk_prov; /* provider name to match */ 139 dtrace_probekey_f *dtpk_pmatch; /* provider matching function */ 140 char *dtpk_mod; /* module name to match */ 141 dtrace_probekey_f *dtpk_mmatch; /* module matching function */ 142 char *dtpk_func; /* func name to match */ 143 dtrace_probekey_f *dtpk_fmatch; /* func matching function */ 144 char *dtpk_name; /* name to match */ 145 dtrace_probekey_f *dtpk_nmatch; /* name matching function */ 146 dtrace_id_t dtpk_id; /* identifier to match */ 147 } dtrace_probekey_t; 148 149 typedef struct dtrace_hashbucket { 150 struct dtrace_hashbucket *dthb_next; /* next on hash chain */ 151 dtrace_probe_t *dthb_chain; /* chain of probes */ 152 int dthb_len; /* number of probes here */ 153 } dtrace_hashbucket_t; 154 155 typedef struct dtrace_hash { 156 dtrace_hashbucket_t **dth_tab; /* hash table */ 157 int dth_size; /* size of hash table */ 158 int dth_mask; /* mask to index into table */ 159 int dth_nbuckets; /* total number of buckets */ 160 uintptr_t dth_nextoffs; /* offset of next in probe */ 161 uintptr_t dth_prevoffs; /* offset of prev in probe */ 162 uintptr_t dth_stroffs; /* offset of str in probe */ 163 } dtrace_hash_t; 164 165 /* 166 * DTrace Enabling Control Blocks 167 * 168 * When a provider wishes to fire a probe, it calls into dtrace_probe(), 169 * passing the probe identifier as the first argument. As described above, 170 * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t 171 * structure. This structure contains information about the probe, and a 172 * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to 173 * DTrace consumer state, and contains an optional predicate, and a list of 174 * actions. (Shown schematically below.) The ECB abstraction allows a single 175 * probe to be multiplexed across disjoint consumers, or across disjoint 176 * enablings of a single probe within one consumer. 177 * 178 * Enabling Control Block 179 * dtrace_ecb_t 180 * +------------------------+ 181 * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID) 182 * | dtrace_state_t * ------+--------------> State associated with this ECB 183 * | dtrace_predicate_t * --+---------+ 184 * | dtrace_action_t * -----+----+ | 185 * | dtrace_ecb_t * ---+ | | | Predicate (if any) 186 * +-------------------+----+ | | dtrace_predicate_t 187 * | | +---> +--------------------+ 188 * | | | dtrace_difo_t * ---+----> DIFO 189 * | | +--------------------+ 190 * | | 191 * Next ECB | | Action 192 * (if any) | | dtrace_action_t 193 * : +--> +-------------------+ 194 * : | dtrace_actkind_t -+------> kind 195 * v | dtrace_difo_t * --+------> DIFO (if any) 196 * | dtrace_recdesc_t -+------> record descr. 197 * | dtrace_action_t * +------+ 198 * +-------------------+ | 199 * | Next action 200 * +-------------------------------+ (if any) 201 * | 202 * | Action 203 * | dtrace_action_t 204 * +--> +-------------------+ 205 * | dtrace_actkind_t -+------> kind 206 * | dtrace_difo_t * --+------> DIFO (if any) 207 * | dtrace_action_t * +------+ 208 * +-------------------+ | 209 * | Next action 210 * +-------------------------------+ (if any) 211 * | 212 * : 213 * v 214 * 215 * 216 * dtrace_probe() iterates over the ECB list. If the ECB needs less space 217 * than is available in the principal buffer, the ECB is processed: if the 218 * predicate is non-NULL, the DIF object is executed. If the result is 219 * non-zero, the action list is processed, with each action being executed 220 * accordingly. When the action list has been completely executed, processing 221 * advances to the next ECB. The ECB abstraction allows disjoint consumers 222 * to multiplex on single probes. 223 * 224 * Execution of the ECB results in consuming dte_size bytes in the buffer 225 * to record data. During execution, dte_needed bytes must be available in 226 * the buffer. This space is used for both recorded data and tuple data. 227 */ 228 struct dtrace_ecb { 229 dtrace_epid_t dte_epid; /* enabled probe ID */ 230 uint32_t dte_alignment; /* required alignment */ 231 size_t dte_needed; /* space needed for execution */ 232 size_t dte_size; /* size of recorded payload */ 233 dtrace_predicate_t *dte_predicate; /* predicate, if any */ 234 dtrace_action_t *dte_action; /* actions, if any */ 235 dtrace_ecb_t *dte_next; /* next ECB on probe */ 236 dtrace_state_t *dte_state; /* pointer to state */ 237 uint32_t dte_cond; /* security condition */ 238 dtrace_probe_t *dte_probe; /* pointer to probe */ 239 dtrace_action_t *dte_action_last; /* last action on ECB */ 240 uint64_t dte_uarg; /* library argument */ 241 }; 242 243 struct dtrace_predicate { 244 dtrace_difo_t *dtp_difo; /* DIF object */ 245 dtrace_cacheid_t dtp_cacheid; /* cache identifier */ 246 int dtp_refcnt; /* reference count */ 247 }; 248 249 struct dtrace_action { 250 dtrace_actkind_t dta_kind; /* kind of action */ 251 uint16_t dta_intuple; /* boolean: in aggregation */ 252 uint32_t dta_refcnt; /* reference count */ 253 dtrace_difo_t *dta_difo; /* pointer to DIFO */ 254 dtrace_recdesc_t dta_rec; /* record description */ 255 dtrace_action_t *dta_prev; /* previous action */ 256 dtrace_action_t *dta_next; /* next action */ 257 }; 258 259 typedef struct dtrace_aggregation { 260 dtrace_action_t dtag_action; /* action; must be first */ 261 dtrace_aggid_t dtag_id; /* identifier */ 262 dtrace_ecb_t *dtag_ecb; /* corresponding ECB */ 263 dtrace_action_t *dtag_first; /* first action in tuple */ 264 uint32_t dtag_base; /* base of aggregation */ 265 uint8_t dtag_hasarg; /* boolean: has argument */ 266 uint64_t dtag_initial; /* initial value */ 267 void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t); 268 } dtrace_aggregation_t; 269 270 /* 271 * DTrace Buffers 272 * 273 * Principal buffers, aggregation buffers, and speculative buffers are all 274 * managed with the dtrace_buffer structure. By default, this structure 275 * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the 276 * active and passive buffers, respectively. For speculative buffers, 277 * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point 278 * to a scratch buffer. For all buffer types, the dtrace_buffer structure is 279 * always allocated on a per-CPU basis; a single dtrace_buffer structure is 280 * never shared among CPUs. (That is, there is never true sharing of the 281 * dtrace_buffer structure; to prevent false sharing of the structure, it must 282 * always be aligned to the coherence granularity -- generally 64 bytes.) 283 * 284 * One of the critical design decisions of DTrace is that a given ECB always 285 * stores the same quantity and type of data. This is done to assure that the 286 * only metadata required for an ECB's traced data is the EPID. That is, from 287 * the EPID, the consumer can determine the data layout. (The data buffer 288 * layout is shown schematically below.) By assuring that one can determine 289 * data layout from the EPID, the metadata stream can be separated from the 290 * data stream -- simplifying the data stream enormously. The ECB always 291 * proceeds the recorded data as part of the dtrace_rechdr_t structure that 292 * includes the EPID and a high-resolution timestamp used for output ordering 293 * consistency. 294 * 295 * base of data buffer ---> +--------+--------------------+--------+ 296 * | rechdr | data | rechdr | 297 * +--------+------+--------+----+--------+ 298 * | data | rechdr | data | 299 * +---------------+--------+-------------+ 300 * | data, cont. | 301 * +--------+--------------------+--------+ 302 * | rechdr | data | | 303 * +--------+--------------------+ | 304 * | || | 305 * | || | 306 * | \/ | 307 * : : 308 * . . 309 * . . 310 * . . 311 * : : 312 * | | 313 * limit of data buffer ---> +--------------------------------------+ 314 * 315 * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the 316 * principal buffer (both scratch and payload) exceed the available space. If 317 * the ECB's needs exceed available space (and if the principal buffer policy 318 * is the default "switch" policy), the ECB is dropped, the buffer's drop count 319 * is incremented, and processing advances to the next ECB. If the ECB's needs 320 * can be met with the available space, the ECB is processed, but the offset in 321 * the principal buffer is only advanced if the ECB completes processing 322 * without error. 323 * 324 * When a buffer is to be switched (either because the buffer is the principal 325 * buffer with a "switch" policy or because it is an aggregation buffer), a 326 * cross call is issued to the CPU associated with the buffer. In the cross 327 * call context, interrupts are disabled, and the active and the inactive 328 * buffers are atomically switched. This involves switching the data pointers, 329 * copying the various state fields (offset, drops, errors, etc.) into their 330 * inactive equivalents, and clearing the state fields. Because interrupts are 331 * disabled during this procedure, the switch is guaranteed to appear atomic to 332 * dtrace_probe(). 333 * 334 * DTrace Ring Buffering 335 * 336 * To process a ring buffer correctly, one must know the oldest valid record. 337 * Processing starts at the oldest record in the buffer and continues until 338 * the end of the buffer is reached. Processing then resumes starting with 339 * the record stored at offset 0 in the buffer, and continues until the 340 * youngest record is processed. If trace records are of a fixed-length, 341 * determining the oldest record is trivial: 342 * 343 * - If the ring buffer has not wrapped, the oldest record is the record 344 * stored at offset 0. 345 * 346 * - If the ring buffer has wrapped, the oldest record is the record stored 347 * at the current offset. 348 * 349 * With variable length records, however, just knowing the current offset 350 * doesn't suffice for determining the oldest valid record: assuming that one 351 * allows for arbitrary data, one has no way of searching forward from the 352 * current offset to find the oldest valid record. (That is, one has no way 353 * of separating data from metadata.) It would be possible to simply refuse to 354 * process any data in the ring buffer between the current offset and the 355 * limit, but this leaves (potentially) an enormous amount of otherwise valid 356 * data unprocessed. 357 * 358 * To effect ring buffering, we track two offsets in the buffer: the current 359 * offset and the _wrapped_ offset. If a request is made to reserve some 360 * amount of data, and the buffer has wrapped, the wrapped offset is 361 * incremented until the wrapped offset minus the current offset is greater 362 * than or equal to the reserve request. This is done by repeatedly looking 363 * up the ECB corresponding to the EPID at the current wrapped offset, and 364 * incrementing the wrapped offset by the size of the data payload 365 * corresponding to that ECB. If this offset is greater than or equal to the 366 * limit of the data buffer, the wrapped offset is set to 0. Thus, the 367 * current offset effectively "chases" the wrapped offset around the buffer. 368 * Schematically: 369 * 370 * base of data buffer ---> +------+--------------------+------+ 371 * | EPID | data | EPID | 372 * +------+--------+------+----+------+ 373 * | data | EPID | data | 374 * +---------------+------+-----------+ 375 * | data, cont. | 376 * +------+---------------------------+ 377 * | EPID | data | 378 * current offset ---> +------+---------------------------+ 379 * | invalid data | 380 * wrapped offset ---> +------+--------------------+------+ 381 * | EPID | data | EPID | 382 * +------+--------+------+----+------+ 383 * | data | EPID | data | 384 * +---------------+------+-----------+ 385 * : : 386 * . . 387 * . ... valid data ... . 388 * . . 389 * : : 390 * +------+-------------+------+------+ 391 * | EPID | data | EPID | data | 392 * +------+------------++------+------+ 393 * | data, cont. | leftover | 394 * limit of data buffer ---> +-------------------+--------------+ 395 * 396 * If the amount of requested buffer space exceeds the amount of space 397 * available between the current offset and the end of the buffer: 398 * 399 * (1) all words in the data buffer between the current offset and the limit 400 * of the data buffer (marked "leftover", above) are set to 401 * DTRACE_EPIDNONE 402 * 403 * (2) the wrapped offset is set to zero 404 * 405 * (3) the iteration process described above occurs until the wrapped offset 406 * is greater than the amount of desired space. 407 * 408 * The wrapped offset is implemented by (re-)using the inactive offset. 409 * In a "switch" buffer policy, the inactive offset stores the offset in 410 * the inactive buffer; in a "ring" buffer policy, it stores the wrapped 411 * offset. 412 * 413 * DTrace Scratch Buffering 414 * 415 * Some ECBs may wish to allocate dynamically-sized temporary scratch memory. 416 * To accommodate such requests easily, scratch memory may be allocated in 417 * the buffer beyond the current offset plus the needed memory of the current 418 * ECB. If there isn't sufficient room in the buffer for the requested amount 419 * of scratch space, the allocation fails and an error is generated. Scratch 420 * memory is tracked in the dtrace_mstate_t and is automatically freed when 421 * the ECB ceases processing. Note that ring buffers cannot allocate their 422 * scratch from the principal buffer -- lest they needlessly overwrite older, 423 * valid data. Ring buffers therefore have their own dedicated scratch buffer 424 * from which scratch is allocated. 425 */ 426 #define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */ 427 #define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */ 428 #define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */ 429 #define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */ 430 #define DTRACEBUF_DROPPED 0x0010 /* drops occurred */ 431 #define DTRACEBUF_ERROR 0x0020 /* errors occurred */ 432 #define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */ 433 #define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */ 434 #define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */ 435 436 typedef struct dtrace_buffer { 437 uint64_t dtb_offset; /* current offset in buffer */ 438 uint64_t dtb_size; /* size of buffer */ 439 uint32_t dtb_flags; /* flags */ 440 uint32_t dtb_drops; /* number of drops */ 441 caddr_t dtb_tomax; /* active buffer */ 442 caddr_t dtb_xamot; /* inactive buffer */ 443 uint32_t dtb_xamot_flags; /* inactive flags */ 444 uint32_t dtb_xamot_drops; /* drops in inactive buffer */ 445 uint64_t dtb_xamot_offset; /* offset in inactive buffer */ 446 uint32_t dtb_errors; /* number of errors */ 447 uint32_t dtb_xamot_errors; /* errors in inactive buffer */ 448 #ifndef _LP64 449 uint64_t dtb_pad1; /* pad out to 64 bytes */ 450 #endif 451 uint64_t dtb_switched; /* time of last switch */ 452 uint64_t dtb_interval; /* observed switch interval */ 453 uint64_t dtb_pad2[6]; /* pad to avoid false sharing */ 454 } dtrace_buffer_t; 455 456 /* 457 * DTrace Aggregation Buffers 458 * 459 * Aggregation buffers use much of the same mechanism as described above 460 * ("DTrace Buffers"). However, because an aggregation is fundamentally a 461 * hash, there exists dynamic metadata associated with an aggregation buffer 462 * that is not associated with other kinds of buffers. This aggregation 463 * metadata is _only_ relevant for the in-kernel implementation of 464 * aggregations; it is not actually relevant to user-level consumers. To do 465 * this, we allocate dynamic aggregation data (hash keys and hash buckets) 466 * starting below the _limit_ of the buffer, and we allocate data from the 467 * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the 468 * data is copied out; the metadata is simply discarded. Schematically, 469 * aggregation buffers look like: 470 * 471 * base of data buffer ---> +-------+------+-----------+-------+ 472 * | aggid | key | value | aggid | 473 * +-------+------+-----------+-------+ 474 * | key | 475 * +-------+-------+-----+------------+ 476 * | value | aggid | key | value | 477 * +-------+------++-----+------+-----+ 478 * | aggid | key | value | | 479 * +-------+------+-------------+ | 480 * | || | 481 * | || | 482 * | \/ | 483 * : : 484 * . . 485 * . . 486 * . . 487 * : : 488 * | /\ | 489 * | || +------------+ 490 * | || | | 491 * +---------------------+ | 492 * | hash keys | 493 * | (dtrace_aggkey structures) | 494 * | | 495 * +----------------------------------+ 496 * | hash buckets | 497 * | (dtrace_aggbuffer structure) | 498 * | | 499 * limit of data buffer ---> +----------------------------------+ 500 * 501 * 502 * As implied above, just as we assure that ECBs always store a constant 503 * amount of data, we assure that a given aggregation -- identified by its 504 * aggregation ID -- always stores data of a constant quantity and type. 505 * As with EPIDs, this allows the aggregation ID to serve as the metadata for a 506 * given record. 507 * 508 * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t) 509 * aligned. (If this the structure changes such that this becomes false, an 510 * assertion will fail in dtrace_aggregate().) 511 */ 512 typedef struct dtrace_aggkey { 513 uint32_t dtak_hashval; /* hash value */ 514 uint32_t dtak_action:4; /* action -- 4 bits */ 515 uint32_t dtak_size:28; /* size -- 28 bits */ 516 caddr_t dtak_data; /* data pointer */ 517 struct dtrace_aggkey *dtak_next; /* next in hash chain */ 518 } dtrace_aggkey_t; 519 520 typedef struct dtrace_aggbuffer { 521 uintptr_t dtagb_hashsize; /* number of buckets */ 522 uintptr_t dtagb_free; /* free list of keys */ 523 dtrace_aggkey_t **dtagb_hash; /* hash table */ 524 } dtrace_aggbuffer_t; 525 526 /* 527 * DTrace Speculations 528 * 529 * Speculations have a per-CPU buffer and a global state. Once a speculation 530 * buffer has been comitted or discarded, it cannot be reused until all CPUs 531 * have taken the same action (commit or discard) on their respective 532 * speculative buffer. However, because DTrace probes may execute in arbitrary 533 * context, other CPUs cannot simply be cross-called at probe firing time to 534 * perform the necessary commit or discard. The speculation states thus 535 * optimize for the case that a speculative buffer is only active on one CPU at 536 * the time of a commit() or discard() -- for if this is the case, other CPUs 537 * need not take action, and the speculation is immediately available for 538 * reuse. If the speculation is active on multiple CPUs, it must be 539 * asynchronously cleaned -- potentially leading to a higher rate of dirty 540 * speculative drops. The speculation states are as follows: 541 * 542 * DTRACESPEC_INACTIVE <= Initial state; inactive speculation 543 * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to 544 * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU 545 * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU 546 * DTRACESPEC_COMMITTING <= Currently being commited on one CPU 547 * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs 548 * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs 549 * 550 * The state transition diagram is as follows: 551 * 552 * +----------------------------------------------------------+ 553 * | | 554 * | +------------+ | 555 * | +-------------------| COMMITTING |<-----------------+ | 556 * | | +------------+ | | 557 * | | copied spec. ^ commit() on | | discard() on 558 * | | into principal | active CPU | | active CPU 559 * | | | commit() | | 560 * V V | | | 561 * +----------+ +--------+ +-----------+ 562 * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE | 563 * +----------+ speculation() +--------+ speculate() +-----------+ 564 * ^ ^ | | | 565 * | | | discard() | | 566 * | | asynchronously | discard() on | | speculate() 567 * | | cleaned V inactive CPU | | on inactive 568 * | | +------------+ | | CPU 569 * | +-------------------| DISCARDING |<-----------------+ | 570 * | +------------+ | 571 * | asynchronously ^ | 572 * | copied spec. | discard() | 573 * | into principal +------------------------+ | 574 * | | V 575 * +----------------+ commit() +------------+ 576 * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY | 577 * +----------------+ +------------+ 578 */ 579 typedef enum dtrace_speculation_state { 580 DTRACESPEC_INACTIVE = 0, 581 DTRACESPEC_ACTIVE, 582 DTRACESPEC_ACTIVEONE, 583 DTRACESPEC_ACTIVEMANY, 584 DTRACESPEC_COMMITTING, 585 DTRACESPEC_COMMITTINGMANY, 586 DTRACESPEC_DISCARDING 587 } dtrace_speculation_state_t; 588 589 typedef struct dtrace_speculation { 590 dtrace_speculation_state_t dtsp_state; /* current speculation state */ 591 int dtsp_cleaning; /* non-zero if being cleaned */ 592 dtrace_buffer_t *dtsp_buffer; /* speculative buffer */ 593 } dtrace_speculation_t; 594 595 /* 596 * DTrace Dynamic Variables 597 * 598 * The dynamic variable problem is obviously decomposed into two subproblems: 599 * allocating new dynamic storage, and freeing old dynamic storage. The 600 * presence of the second problem makes the first much more complicated -- or 601 * rather, the absence of the second renders the first trivial. This is the 602 * case with aggregations, for which there is effectively no deallocation of 603 * dynamic storage. (Or more accurately, all dynamic storage is deallocated 604 * when a snapshot is taken of the aggregation.) As DTrace dynamic variables 605 * allow for both dynamic allocation and dynamic deallocation, the 606 * implementation of dynamic variables is quite a bit more complicated than 607 * that of their aggregation kin. 608 * 609 * We observe that allocating new dynamic storage is tricky only because the 610 * size can vary -- the allocation problem is much easier if allocation sizes 611 * are uniform. We further observe that in D, the size of dynamic variables is 612 * actually _not_ dynamic -- dynamic variable sizes may be determined by static 613 * analysis of DIF text. (This is true even of putatively dynamically-sized 614 * objects like strings and stacks, the sizes of which are dictated by the 615 * "stringsize" and "stackframes" variables, respectively.) We exploit this by 616 * performing this analysis on all DIF before enabling any probes. For each 617 * dynamic load or store, we calculate the dynamically-allocated size plus the 618 * size of the dtrace_dynvar structure plus the storage required to key the 619 * data. For all DIF, we take the largest value and dub it the _chunksize_. 620 * We then divide dynamic memory into two parts: a hash table that is wide 621 * enough to have every chunk in its own bucket, and a larger region of equal 622 * chunksize units. Whenever we wish to dynamically allocate a variable, we 623 * always allocate a single chunk of memory. Depending on the uniformity of 624 * allocation, this will waste some amount of memory -- but it eliminates the 625 * non-determinism inherent in traditional heap fragmentation. 626 * 627 * Dynamic objects are allocated by storing a non-zero value to them; they are 628 * deallocated by storing a zero value to them. Dynamic variables are 629 * complicated enormously by being shared between CPUs. In particular, 630 * consider the following scenario: 631 * 632 * CPU A CPU B 633 * +---------------------------------+ +---------------------------------+ 634 * | | | | 635 * | allocates dynamic object a[123] | | | 636 * | by storing the value 345 to it | | | 637 * | ---------> | 638 * | | | wishing to load from object | 639 * | | | a[123], performs lookup in | 640 * | | | dynamic variable space | 641 * | <--------- | 642 * | deallocates object a[123] by | | | 643 * | storing 0 to it | | | 644 * | | | | 645 * | allocates dynamic object b[567] | | performs load from a[123] | 646 * | by storing the value 789 to it | | | 647 * : : : : 648 * . . . . 649 * 650 * This is obviously a race in the D program, but there are nonetheless only 651 * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly, 652 * CPU B may _not_ see the value 789 for a[123]. 653 * 654 * There are essentially two ways to deal with this: 655 * 656 * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load 657 * from a[123], it needs to lock a[123] and hold the lock for the 658 * duration that it wishes to manipulate it. 659 * 660 * (2) Avoid reusing freed chunks until it is known that no CPU is referring 661 * to them. 662 * 663 * The implementation of (1) is rife with complexity, because it requires the 664 * user of a dynamic variable to explicitly decree when they are done using it. 665 * Were all variables by value, this perhaps wouldn't be debilitating -- but 666 * dynamic variables of non-scalar types are tracked by reference. That is, if 667 * a dynamic variable is, say, a string, and that variable is to be traced to, 668 * say, the principal buffer, the DIF emulation code returns to the main 669 * dtrace_probe() loop a pointer to the underlying storage, not the contents of 670 * the storage. Further, code calling on DIF emulation would have to be aware 671 * that the DIF emulation has returned a reference to a dynamic variable that 672 * has been potentially locked. The variable would have to be unlocked after 673 * the main dtrace_probe() loop is finished with the variable, and the main 674 * dtrace_probe() loop would have to be careful to not call any further DIF 675 * emulation while the variable is locked to avoid deadlock. More generally, 676 * if one were to implement (1), DIF emulation code dealing with dynamic 677 * variables could only deal with one dynamic variable at a time (lest deadlock 678 * result). To sum, (1) exports too much subtlety to the users of dynamic 679 * variables -- increasing maintenance burden and imposing serious constraints 680 * on future DTrace development. 681 * 682 * The implementation of (2) is also complex, but the complexity is more 683 * manageable. We need to be sure that when a variable is deallocated, it is 684 * not placed on a traditional free list, but rather on a _dirty_ list. Once a 685 * variable is on a dirty list, it cannot be found by CPUs performing a 686 * subsequent lookup of the variable -- but it may still be in use by other 687 * CPUs. To assure that all CPUs that may be seeing the old variable have 688 * cleared out of probe context, a dtrace_sync() can be issued. Once the 689 * dtrace_sync() has completed, it can be known that all CPUs are done 690 * manipulating the dynamic variable -- the dirty list can be atomically 691 * appended to the free list. Unfortunately, there's a slight hiccup in this 692 * mechanism: dtrace_sync() may not be issued from probe context. The 693 * dtrace_sync() must be therefore issued asynchronously from non-probe 694 * context. For this we rely on the DTrace cleaner, a cyclic that runs at the 695 * "cleanrate" frequency. To ease this implementation, we define several chunk 696 * lists: 697 * 698 * - Dirty. Deallocated chunks, not yet cleaned. Not available. 699 * 700 * - Rinsing. Formerly dirty chunks that are currently being asynchronously 701 * cleaned. Not available, but will be shortly. Dynamic variable 702 * allocation may not spin or block for availability, however. 703 * 704 * - Clean. Clean chunks, ready for allocation -- but not on the free list. 705 * 706 * - Free. Available for allocation. 707 * 708 * Moreover, to avoid absurd contention, _each_ of these lists is implemented 709 * on a per-CPU basis. This is only for performance, not correctness; chunks 710 * may be allocated from another CPU's free list. The algorithm for allocation 711 * then is this: 712 * 713 * (1) Attempt to atomically allocate from current CPU's free list. If list 714 * is non-empty and allocation is successful, allocation is complete. 715 * 716 * (2) If the clean list is non-empty, atomically move it to the free list, 717 * and reattempt (1). 718 * 719 * (3) If the dynamic variable space is in the CLEAN state, look for free 720 * and clean lists on other CPUs by setting the current CPU to the next 721 * CPU, and reattempting (1). If the next CPU is the current CPU (that 722 * is, if all CPUs have been checked), atomically switch the state of 723 * the dynamic variable space based on the following: 724 * 725 * - If no free chunks were found and no dirty chunks were found, 726 * atomically set the state to EMPTY. 727 * 728 * - If dirty chunks were found, atomically set the state to DIRTY. 729 * 730 * - If rinsing chunks were found, atomically set the state to RINSING. 731 * 732 * (4) Based on state of dynamic variable space state, increment appropriate 733 * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic 734 * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in 735 * RINSING state). Fail the allocation. 736 * 737 * The cleaning cyclic operates with the following algorithm: for all CPUs 738 * with a non-empty dirty list, atomically move the dirty list to the rinsing 739 * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list, 740 * atomically move the rinsing list to the clean list. Perform another 741 * dtrace_sync(). By this point, all CPUs have seen the new clean list; the 742 * state of the dynamic variable space can be restored to CLEAN. 743 * 744 * There exist two final races that merit explanation. The first is a simple 745 * allocation race: 746 * 747 * CPU A CPU B 748 * +---------------------------------+ +---------------------------------+ 749 * | | | | 750 * | allocates dynamic object a[123] | | allocates dynamic object a[123] | 751 * | by storing the value 345 to it | | by storing the value 567 to it | 752 * | | | | 753 * : : : : 754 * . . . . 755 * 756 * Again, this is a race in the D program. It can be resolved by having a[123] 757 * hold the value 345 or a[123] hold the value 567 -- but it must be true that 758 * a[123] have only _one_ of these values. (That is, the racing CPUs may not 759 * put the same element twice on the same hash chain.) This is resolved 760 * simply: before the allocation is undertaken, the start of the new chunk's 761 * hash chain is noted. Later, after the allocation is complete, the hash 762 * chain is atomically switched to point to the new element. If this fails 763 * (because of either concurrent allocations or an allocation concurrent with a 764 * deletion), the newly allocated chunk is deallocated to the dirty list, and 765 * the whole process of looking up (and potentially allocating) the dynamic 766 * variable is reattempted. 767 * 768 * The final race is a simple deallocation race: 769 * 770 * CPU A CPU B 771 * +---------------------------------+ +---------------------------------+ 772 * | | | | 773 * | deallocates dynamic object | | deallocates dynamic object | 774 * | a[123] by storing the value 0 | | a[123] by storing the value 0 | 775 * | to it | | to it | 776 * | | | | 777 * : : : : 778 * . . . . 779 * 780 * Once again, this is a race in the D program, but it is one that we must 781 * handle without corrupting the underlying data structures. Because 782 * deallocations require the deletion of a chunk from the middle of a hash 783 * chain, we cannot use a single-word atomic operation to remove it. For this, 784 * we add a spin lock to the hash buckets that is _only_ used for deallocations 785 * (allocation races are handled as above). Further, this spin lock is _only_ 786 * held for the duration of the delete; before control is returned to the DIF 787 * emulation code, the hash bucket is unlocked. 788 */ 789 typedef struct dtrace_key { 790 uint64_t dttk_value; /* data value or data pointer */ 791 uint64_t dttk_size; /* 0 if by-val, >0 if by-ref */ 792 } dtrace_key_t; 793 794 typedef struct dtrace_tuple { 795 uint32_t dtt_nkeys; /* number of keys in tuple */ 796 uint32_t dtt_pad; /* padding */ 797 dtrace_key_t dtt_key[1]; /* array of tuple keys */ 798 } dtrace_tuple_t; 799 800 typedef struct dtrace_dynvar { 801 uint64_t dtdv_hashval; /* hash value -- 0 if free */ 802 struct dtrace_dynvar *dtdv_next; /* next on list or hash chain */ 803 void *dtdv_data; /* pointer to data */ 804 dtrace_tuple_t dtdv_tuple; /* tuple key */ 805 } dtrace_dynvar_t; 806 807 typedef enum dtrace_dynvar_op { 808 DTRACE_DYNVAR_ALLOC, 809 DTRACE_DYNVAR_NOALLOC, 810 DTRACE_DYNVAR_DEALLOC 811 } dtrace_dynvar_op_t; 812 813 typedef struct dtrace_dynhash { 814 dtrace_dynvar_t *dtdh_chain; /* hash chain for this bucket */ 815 uintptr_t dtdh_lock; /* deallocation lock */ 816 #ifdef _LP64 817 uintptr_t dtdh_pad[6]; /* pad to avoid false sharing */ 818 #else 819 uintptr_t dtdh_pad[14]; /* pad to avoid false sharing */ 820 #endif 821 } dtrace_dynhash_t; 822 823 typedef struct dtrace_dstate_percpu { 824 dtrace_dynvar_t *dtdsc_free; /* free list for this CPU */ 825 dtrace_dynvar_t *dtdsc_dirty; /* dirty list for this CPU */ 826 dtrace_dynvar_t *dtdsc_rinsing; /* rinsing list for this CPU */ 827 dtrace_dynvar_t *dtdsc_clean; /* clean list for this CPU */ 828 uint64_t dtdsc_drops; /* number of capacity drops */ 829 uint64_t dtdsc_dirty_drops; /* number of dirty drops */ 830 uint64_t dtdsc_rinsing_drops; /* number of rinsing drops */ 831 #ifdef _LP64 832 uint64_t dtdsc_pad; /* pad to avoid false sharing */ 833 #else 834 uint64_t dtdsc_pad[2]; /* pad to avoid false sharing */ 835 #endif 836 } dtrace_dstate_percpu_t; 837 838 typedef enum dtrace_dstate_state { 839 DTRACE_DSTATE_CLEAN = 0, 840 DTRACE_DSTATE_EMPTY, 841 DTRACE_DSTATE_DIRTY, 842 DTRACE_DSTATE_RINSING 843 } dtrace_dstate_state_t; 844 845 typedef struct dtrace_dstate { 846 void *dtds_base; /* base of dynamic var. space */ 847 size_t dtds_size; /* size of dynamic var. space */ 848 size_t dtds_hashsize; /* number of buckets in hash */ 849 size_t dtds_chunksize; /* size of each chunk */ 850 dtrace_dynhash_t *dtds_hash; /* pointer to hash table */ 851 dtrace_dstate_state_t dtds_state; /* current dynamic var. state */ 852 dtrace_dstate_percpu_t *dtds_percpu; /* per-CPU dyn. var. state */ 853 } dtrace_dstate_t; 854 855 /* 856 * DTrace Variable State 857 * 858 * The DTrace variable state tracks user-defined variables in its dtrace_vstate 859 * structure. Each DTrace consumer has exactly one dtrace_vstate structure, 860 * but some dtrace_vstate structures may exist without a corresponding DTrace 861 * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>, 862 * user-defined variables can have one of three scopes: 863 * 864 * DIFV_SCOPE_GLOBAL => global scope 865 * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables) 866 * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables) 867 * 868 * The variable state tracks variables by both their scope and their allocation 869 * type: 870 * 871 * - The dtvs_globals and dtvs_locals members each point to an array of 872 * dtrace_statvar structures. These structures contain both the variable 873 * metadata (dtrace_difv structures) and the underlying storage for all 874 * statically allocated variables, including statically allocated 875 * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables. 876 * 877 * - The dtvs_tlocals member points to an array of dtrace_difv structures for 878 * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the 879 * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage 880 * is allocated out of the dynamic variable space. 881 * 882 * - The dtvs_dynvars member is the dynamic variable state associated with the 883 * variable state. The dynamic variable state (described in "DTrace Dynamic 884 * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all 885 * dynamically-allocated DIFV_SCOPE_GLOBAL variables. 886 */ 887 typedef struct dtrace_statvar { 888 uint64_t dtsv_data; /* data or pointer to it */ 889 size_t dtsv_size; /* size of pointed-to data */ 890 int dtsv_refcnt; /* reference count */ 891 dtrace_difv_t dtsv_var; /* variable metadata */ 892 } dtrace_statvar_t; 893 894 typedef struct dtrace_vstate { 895 dtrace_state_t *dtvs_state; /* back pointer to state */ 896 dtrace_statvar_t **dtvs_globals; /* statically-allocated glbls */ 897 int dtvs_nglobals; /* number of globals */ 898 dtrace_difv_t *dtvs_tlocals; /* thread-local metadata */ 899 int dtvs_ntlocals; /* number of thread-locals */ 900 dtrace_statvar_t **dtvs_locals; /* clause-local data */ 901 int dtvs_nlocals; /* number of clause-locals */ 902 dtrace_dstate_t dtvs_dynvars; /* dynamic variable state */ 903 } dtrace_vstate_t; 904 905 /* 906 * DTrace Machine State 907 * 908 * In the process of processing a fired probe, DTrace needs to track and/or 909 * cache some per-CPU state associated with that particular firing. This is 910 * state that is always discarded after the probe firing has completed, and 911 * much of it is not specific to any DTrace consumer, remaining valid across 912 * all ECBs. This state is tracked in the dtrace_mstate structure. 913 */ 914 #define DTRACE_MSTATE_ARGS 0x00000001 915 #define DTRACE_MSTATE_PROBE 0x00000002 916 #define DTRACE_MSTATE_EPID 0x00000004 917 #define DTRACE_MSTATE_TIMESTAMP 0x00000008 918 #define DTRACE_MSTATE_STACKDEPTH 0x00000010 919 #define DTRACE_MSTATE_CALLER 0x00000020 920 #define DTRACE_MSTATE_IPL 0x00000040 921 #define DTRACE_MSTATE_FLTOFFS 0x00000080 922 #define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100 923 #define DTRACE_MSTATE_USTACKDEPTH 0x00000200 924 #define DTRACE_MSTATE_UCALLER 0x00000400 925 926 typedef struct dtrace_mstate { 927 uintptr_t dtms_scratch_base; /* base of scratch space */ 928 uintptr_t dtms_scratch_ptr; /* current scratch pointer */ 929 size_t dtms_scratch_size; /* scratch size */ 930 uint32_t dtms_present; /* variables that are present */ 931 uint64_t dtms_arg[5]; /* cached arguments */ 932 dtrace_epid_t dtms_epid; /* current EPID */ 933 uint64_t dtms_timestamp; /* cached timestamp */ 934 hrtime_t dtms_walltimestamp; /* cached wall timestamp */ 935 int dtms_stackdepth; /* cached stackdepth */ 936 int dtms_ustackdepth; /* cached ustackdepth */ 937 struct dtrace_probe *dtms_probe; /* current probe */ 938 uintptr_t dtms_caller; /* cached caller */ 939 uint64_t dtms_ucaller; /* cached user-level caller */ 940 int dtms_ipl; /* cached interrupt pri lev */ 941 int dtms_fltoffs; /* faulting DIFO offset */ 942 uintptr_t dtms_strtok; /* saved strtok() pointer */ 943 uintptr_t dtms_strtok_limit; /* upper bound of strtok ptr */ 944 uint32_t dtms_access; /* memory access rights */ 945 dtrace_difo_t *dtms_difo; /* current dif object */ 946 file_t *dtms_getf; /* cached rval of getf() */ 947 } dtrace_mstate_t; 948 949 #define DTRACE_COND_OWNER 0x1 950 #define DTRACE_COND_USERMODE 0x2 951 #define DTRACE_COND_ZONEOWNER 0x4 952 953 #define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */ 954 955 /* 956 * Access flag used by dtrace_mstate.dtms_access. 957 */ 958 #define DTRACE_ACCESS_KERNEL 0x1 /* the priv to read kmem */ 959 960 961 /* 962 * DTrace Activity 963 * 964 * Each DTrace consumer is in one of several states, which (for purposes of 965 * avoiding yet-another overloading of the noun "state") we call the current 966 * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on 967 * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may 968 * only transition in one direction; the activity transition diagram is a 969 * directed acyclic graph. The activity transition diagram is as follows: 970 * 971 * 972 * +----------+ +--------+ +--------+ 973 * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE | 974 * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+ 975 * before BEGIN | after BEGIN | | | 976 * | | | | 977 * exit() action | | | | 978 * from BEGIN ECB | | | | 979 * | | | | 980 * v | | | 981 * +----------+ exit() action | | | 982 * +-----------------------------| DRAINING |<-------------------+ | | 983 * | +----------+ | | 984 * | | | | 985 * | dtrace_stop(), | | | 986 * | before END | | | 987 * | | | | 988 * | v | | 989 * | +---------+ +----------+ | | 990 * | | STOPPED |<----------------| COOLDOWN |<----------------------+ | 991 * | +---------+ dtrace_stop(), +----------+ dtrace_stop(), | 992 * | after END before END | 993 * | | 994 * | +--------+ | 995 * +----------------------------->| KILLED |<--------------------------+ 996 * deadman timeout or +--------+ deadman timeout or 997 * killed consumer killed consumer 998 * 999 * Note that once a DTrace consumer has stopped tracing, there is no way to 1000 * restart it; if a DTrace consumer wishes to restart tracing, it must reopen 1001 * the DTrace pseudodevice. 1002 */ 1003 typedef enum dtrace_activity { 1004 DTRACE_ACTIVITY_INACTIVE = 0, /* not yet running */ 1005 DTRACE_ACTIVITY_WARMUP, /* while starting */ 1006 DTRACE_ACTIVITY_ACTIVE, /* running */ 1007 DTRACE_ACTIVITY_DRAINING, /* before stopping */ 1008 DTRACE_ACTIVITY_COOLDOWN, /* while stopping */ 1009 DTRACE_ACTIVITY_STOPPED, /* after stopping */ 1010 DTRACE_ACTIVITY_KILLED /* killed */ 1011 } dtrace_activity_t; 1012 1013 /* 1014 * DTrace Helper Implementation 1015 * 1016 * A description of the helper architecture may be found in <sys/dtrace.h>. 1017 * Each process contains a pointer to its helpers in its p_dtrace_helpers 1018 * member. This is a pointer to a dtrace_helpers structure, which contains an 1019 * array of pointers to dtrace_helper structures, helper variable state (shared 1020 * among a process's helpers) and a generation count. (The generation count is 1021 * used to provide an identifier when a helper is added so that it may be 1022 * subsequently removed.) The dtrace_helper structure is self-explanatory, 1023 * containing pointers to the objects needed to execute the helper. Note that 1024 * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more 1025 * than dtrace_helpers_max are allowed per-process. 1026 */ 1027 #define DTRACE_HELPER_ACTION_USTACK 0 1028 #define DTRACE_NHELPER_ACTIONS 1 1029 1030 typedef struct dtrace_helper_action { 1031 int dtha_generation; /* helper action generation */ 1032 int dtha_nactions; /* number of actions */ 1033 dtrace_difo_t *dtha_predicate; /* helper action predicate */ 1034 dtrace_difo_t **dtha_actions; /* array of actions */ 1035 struct dtrace_helper_action *dtha_next; /* next helper action */ 1036 } dtrace_helper_action_t; 1037 1038 typedef struct dtrace_helper_provider { 1039 int dthp_generation; /* helper provider generation */ 1040 uint32_t dthp_ref; /* reference count */ 1041 dof_helper_t dthp_prov; /* DOF w/ provider and probes */ 1042 } dtrace_helper_provider_t; 1043 1044 typedef struct dtrace_helpers { 1045 dtrace_helper_action_t **dthps_actions; /* array of helper actions */ 1046 dtrace_vstate_t dthps_vstate; /* helper action var. state */ 1047 dtrace_helper_provider_t **dthps_provs; /* array of providers */ 1048 uint_t dthps_nprovs; /* count of providers */ 1049 uint_t dthps_maxprovs; /* provider array size */ 1050 int dthps_generation; /* current generation */ 1051 pid_t dthps_pid; /* pid of associated proc */ 1052 int dthps_deferred; /* helper in deferred list */ 1053 struct dtrace_helpers *dthps_next; /* next pointer */ 1054 struct dtrace_helpers *dthps_prev; /* prev pointer */ 1055 } dtrace_helpers_t; 1056 1057 /* 1058 * DTrace Helper Action Tracing 1059 * 1060 * Debugging helper actions can be arduous. To ease the development and 1061 * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing- 1062 * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which 1063 * it is by default on DEBUG kernels), all helper activity will be traced to a 1064 * global, in-kernel ring buffer. Each entry includes a pointer to the specific 1065 * helper, the location within the helper, and a trace of all local variables. 1066 * The ring buffer may be displayed in a human-readable format with the 1067 * ::dtrace_helptrace mdb(1) dcmd. 1068 */ 1069 #define DTRACE_HELPTRACE_NEXT (-1) 1070 #define DTRACE_HELPTRACE_DONE (-2) 1071 #define DTRACE_HELPTRACE_ERR (-3) 1072 1073 typedef struct dtrace_helptrace { 1074 dtrace_helper_action_t *dtht_helper; /* helper action */ 1075 int dtht_where; /* where in helper action */ 1076 int dtht_nlocals; /* number of locals */ 1077 int dtht_fault; /* type of fault (if any) */ 1078 int dtht_fltoffs; /* DIF offset */ 1079 uint64_t dtht_illval; /* faulting value */ 1080 uint64_t dtht_locals[1]; /* local variables */ 1081 } dtrace_helptrace_t; 1082 1083 /* 1084 * DTrace Credentials 1085 * 1086 * In probe context, we have limited flexibility to examine the credentials 1087 * of the DTrace consumer that created a particular enabling. We use 1088 * the Least Privilege interfaces to cache the consumer's cred pointer and 1089 * some facts about that credential in a dtrace_cred_t structure. These 1090 * can limit the consumer's breadth of visibility and what actions the 1091 * consumer may take. 1092 */ 1093 #define DTRACE_CRV_ALLPROC 0x01 1094 #define DTRACE_CRV_KERNEL 0x02 1095 #define DTRACE_CRV_ALLZONE 0x04 1096 1097 #define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \ 1098 DTRACE_CRV_ALLZONE) 1099 1100 #define DTRACE_CRA_PROC 0x0001 1101 #define DTRACE_CRA_PROC_CONTROL 0x0002 1102 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER 0x0004 1103 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE 0x0008 1104 #define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG 0x0010 1105 #define DTRACE_CRA_KERNEL 0x0020 1106 #define DTRACE_CRA_KERNEL_DESTRUCTIVE 0x0040 1107 1108 #define DTRACE_CRA_ALL (DTRACE_CRA_PROC | \ 1109 DTRACE_CRA_PROC_CONTROL | \ 1110 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \ 1111 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \ 1112 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \ 1113 DTRACE_CRA_KERNEL | \ 1114 DTRACE_CRA_KERNEL_DESTRUCTIVE) 1115 1116 typedef struct dtrace_cred { 1117 cred_t *dcr_cred; 1118 uint8_t dcr_destructive; 1119 uint8_t dcr_visible; 1120 uint16_t dcr_action; 1121 } dtrace_cred_t; 1122 1123 /* 1124 * DTrace Consumer State 1125 * 1126 * Each DTrace consumer has an associated dtrace_state structure that contains 1127 * its in-kernel DTrace state -- including options, credentials, statistics and 1128 * pointers to ECBs, buffers, speculations and formats. A dtrace_state 1129 * structure is also allocated for anonymous enablings. When anonymous state 1130 * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed 1131 * dtrace_state structure. 1132 */ 1133 struct dtrace_state { 1134 #ifdef __FreeBSD__ 1135 struct cdev *dts_dev; /* device */ 1136 #else 1137 dev_t dts_dev; /* device */ 1138 #endif 1139 int dts_necbs; /* total number of ECBs */ 1140 dtrace_ecb_t **dts_ecbs; /* array of ECBs */ 1141 dtrace_epid_t dts_epid; /* next EPID to allocate */ 1142 size_t dts_needed; /* greatest needed space */ 1143 struct dtrace_state *dts_anon; /* anon. state, if grabbed */ 1144 dtrace_activity_t dts_activity; /* current activity */ 1145 dtrace_vstate_t dts_vstate; /* variable state */ 1146 dtrace_buffer_t *dts_buffer; /* principal buffer */ 1147 dtrace_buffer_t *dts_aggbuffer; /* aggregation buffer */ 1148 dtrace_speculation_t *dts_speculations; /* speculation array */ 1149 int dts_nspeculations; /* number of speculations */ 1150 int dts_naggregations; /* number of aggregations */ 1151 dtrace_aggregation_t **dts_aggregations; /* aggregation array */ 1152 #ifdef __FreeBSD__ 1153 struct unrhdr *dts_aggid_arena; /* arena for aggregation IDs */ 1154 #else 1155 vmem_t *dts_aggid_arena; /* arena for aggregation IDs */ 1156 #endif 1157 uint64_t dts_errors; /* total number of errors */ 1158 uint32_t dts_speculations_busy; /* number of spec. busy */ 1159 uint32_t dts_speculations_unavail; /* number of spec unavail */ 1160 uint32_t dts_stkstroverflows; /* stack string tab overflows */ 1161 uint32_t dts_dblerrors; /* errors in ERROR probes */ 1162 uint32_t dts_reserve; /* space reserved for END */ 1163 hrtime_t dts_laststatus; /* time of last status */ 1164 #ifdef illumos 1165 cyclic_id_t dts_cleaner; /* cleaning cyclic */ 1166 cyclic_id_t dts_deadman; /* deadman cyclic */ 1167 #endif 1168 #ifdef __FreeBSD__ 1169 struct callout dts_cleaner; /* Cleaning callout. */ 1170 struct callout dts_deadman; /* Deadman callout. */ 1171 #endif 1172 #ifdef __NetBSD__ 1173 struct dtrace_state_worker *dts_cleaner;/* cleaning cyclic */ 1174 struct dtrace_state_worker *dts_deadman;/* deadman cyclic */ 1175 #endif 1176 hrtime_t dts_alive; /* time last alive */ 1177 char dts_speculates; /* boolean: has speculations */ 1178 char dts_destructive; /* boolean: has dest. actions */ 1179 int dts_nformats; /* number of formats */ 1180 char **dts_formats; /* format string array */ 1181 dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */ 1182 dtrace_cred_t dts_cred; /* credentials */ 1183 size_t dts_nretained; /* number of retained enabs */ 1184 int dts_getf; /* number of getf() calls */ 1185 uint64_t dts_rstate[NCPU][2]; /* per-CPU random state */ 1186 }; 1187 1188 struct dtrace_provider { 1189 dtrace_pattr_t dtpv_attr; /* provider attributes */ 1190 dtrace_ppriv_t dtpv_priv; /* provider privileges */ 1191 dtrace_pops_t dtpv_pops; /* provider operations */ 1192 char *dtpv_name; /* provider name */ 1193 void *dtpv_arg; /* provider argument */ 1194 hrtime_t dtpv_defunct; /* when made defunct */ 1195 struct dtrace_provider *dtpv_next; /* next provider */ 1196 }; 1197 1198 struct dtrace_meta { 1199 dtrace_mops_t dtm_mops; /* meta provider operations */ 1200 char *dtm_name; /* meta provider name */ 1201 void *dtm_arg; /* meta provider user arg */ 1202 uint64_t dtm_count; /* no. of associated provs. */ 1203 }; 1204 1205 /* 1206 * DTrace Enablings 1207 * 1208 * A dtrace_enabling structure is used to track a collection of ECB 1209 * descriptions -- before they have been turned into actual ECBs. This is 1210 * created as a result of DOF processing, and is generally used to generate 1211 * ECBs immediately thereafter. However, enablings are also generally 1212 * retained should the probes they describe be created at a later time; as 1213 * each new module or provider registers with the framework, the retained 1214 * enablings are reevaluated, with any new match resulting in new ECBs. To 1215 * prevent probes from being matched more than once, the enabling tracks the 1216 * last probe generation matched, and only matches probes from subsequent 1217 * generations. 1218 */ 1219 typedef struct dtrace_enabling { 1220 dtrace_ecbdesc_t **dten_desc; /* all ECB descriptions */ 1221 int dten_ndesc; /* number of ECB descriptions */ 1222 int dten_maxdesc; /* size of ECB array */ 1223 dtrace_vstate_t *dten_vstate; /* associated variable state */ 1224 dtrace_genid_t dten_probegen; /* matched probe generation */ 1225 dtrace_ecbdesc_t *dten_current; /* current ECB description */ 1226 int dten_error; /* current error value */ 1227 int dten_primed; /* boolean: set if primed */ 1228 struct dtrace_enabling *dten_prev; /* previous enabling */ 1229 struct dtrace_enabling *dten_next; /* next enabling */ 1230 } dtrace_enabling_t; 1231 1232 /* 1233 * DTrace Anonymous Enablings 1234 * 1235 * Anonymous enablings are DTrace enablings that are not associated with a 1236 * controlling process, but rather derive their enabling from DOF stored as 1237 * properties in the dtrace.conf file. If there is an anonymous enabling, a 1238 * DTrace consumer state and enabling are created on attach. The state may be 1239 * subsequently grabbed by the first consumer specifying the "grabanon" 1240 * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will 1241 * refuse to unload. 1242 */ 1243 typedef struct dtrace_anon { 1244 dtrace_state_t *dta_state; /* DTrace consumer state */ 1245 dtrace_enabling_t *dta_enabling; /* pointer to enabling */ 1246 processorid_t dta_beganon; /* which CPU BEGIN ran on */ 1247 } dtrace_anon_t; 1248 1249 /* 1250 * DTrace Error Debugging 1251 */ 1252 #ifdef DEBUG 1253 #define DTRACE_ERRDEBUG 1254 #endif 1255 1256 #ifdef DTRACE_ERRDEBUG 1257 1258 typedef struct dtrace_errhash { 1259 const char *dter_msg; /* error message */ 1260 int dter_count; /* number of times seen */ 1261 } dtrace_errhash_t; 1262 1263 #define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */ 1264 1265 #endif /* DTRACE_ERRDEBUG */ 1266 1267 /* 1268 * DTrace Toxic Ranges 1269 * 1270 * DTrace supports safe loads from probe context; if the address turns out to 1271 * be invalid, a bit will be set by the kernel indicating that DTrace 1272 * encountered a memory error, and DTrace will propagate the error to the user 1273 * accordingly. However, there may exist some regions of memory in which an 1274 * arbitrary load can change system state, and from which it is impossible to 1275 * recover from such a load after it has been attempted. Examples of this may 1276 * include memory in which programmable I/O registers are mapped (for which a 1277 * read may have some implications for the device) or (in the specific case of 1278 * UltraSPARC-I and -II) the virtual address hole. The platform is required 1279 * to make DTrace aware of these toxic ranges; DTrace will then check that 1280 * target addresses are not in a toxic range before attempting to issue a 1281 * safe load. 1282 */ 1283 typedef struct dtrace_toxrange { 1284 uintptr_t dtt_base; /* base of toxic range */ 1285 uintptr_t dtt_limit; /* limit of toxic range */ 1286 } dtrace_toxrange_t; 1287 1288 #ifdef illumos 1289 extern uint64_t dtrace_getarg(int, int); 1290 #else 1291 extern uint64_t __noinline dtrace_getarg(int, int); 1292 #endif 1293 extern greg_t dtrace_getfp(void); 1294 extern int dtrace_getipl(void); 1295 extern uintptr_t dtrace_caller(int); 1296 extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t); 1297 extern void *dtrace_casptr(volatile void *, volatile void *, volatile void *); 1298 extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1299 extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1300 extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1301 extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t, 1302 volatile uint16_t *); 1303 extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *); 1304 extern ulong_t dtrace_getreg(struct trapframe *, uint_t); 1305 extern int dtrace_getstackdepth(int); 1306 extern void dtrace_getupcstack(uint64_t *, int); 1307 extern void dtrace_getufpstack(uint64_t *, uint64_t *, int); 1308 extern int dtrace_getustackdepth(void); 1309 extern uintptr_t dtrace_fulword(void *); 1310 extern uint8_t dtrace_fuword8(void *); 1311 extern uint16_t dtrace_fuword16(void *); 1312 extern uint32_t dtrace_fuword32(void *); 1313 extern uint64_t dtrace_fuword64(void *); 1314 extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int, 1315 int, uintptr_t); 1316 extern int dtrace_assfail(const char *, const char *, int); 1317 extern int dtrace_attached(void); 1318 #ifdef illumos 1319 extern hrtime_t dtrace_gethrestime(void); 1320 #endif 1321 1322 #ifdef __sparc 1323 extern void dtrace_flush_windows(void); 1324 extern void dtrace_flush_user_windows(void); 1325 extern uint_t dtrace_getotherwin(void); 1326 extern uint_t dtrace_getfprs(void); 1327 #else 1328 extern void dtrace_copy(uintptr_t, uintptr_t, size_t); 1329 extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1330 #endif 1331 1332 /* 1333 * DTrace Assertions 1334 * 1335 * DTrace calls ASSERT and VERIFY from probe context. To assure that a failed 1336 * ASSERT or VERIFY does not induce a markedly more catastrophic failure (e.g., 1337 * one from which a dump cannot be gleaned), DTrace must define its own ASSERT 1338 * and VERIFY macros to be ones that may safely be called from probe context. 1339 * This header file must thus be included by any DTrace component that calls 1340 * ASSERT and/or VERIFY from probe context, and _only_ by those components. 1341 * (The only exception to this is kernel debugging infrastructure at user-level 1342 * that doesn't depend on calling ASSERT.) 1343 */ 1344 #undef ASSERT 1345 #undef VERIFY 1346 #define VERIFY(EX) ((void)((EX) || \ 1347 dtrace_assfail(#EX, __FILE__, __LINE__))) 1348 #ifdef DEBUG 1349 #define ASSERT(EX) ((void)((EX) || \ 1350 dtrace_assfail(#EX, __FILE__, __LINE__))) 1351 #else 1352 #define ASSERT(X) ((void)0) 1353 #endif 1354 1355 #ifdef __cplusplus 1356 } 1357 #endif 1358 1359 #endif /* _SYS_DTRACE_IMPL_H */ 1360